Browse Source
Prometheus endpoint (#5256)
Prometheus endpoint (#5256)
* Add prometheus collector and route * dep ensure -add github.com/prometheus/client_golang/prometheus * dep ensure -update github.com/golang/protobuf * add metrics to reserved usernames * add comment head in metrics package * fix style imports * add metrics settings * add bearer token check * mapping metrics configs * fix lint * update config cheat sheet * update conf sample, typo fixrelease/v1.7
committed by
techknowlogick
112 changed files with 24655 additions and 2973 deletions
-
68Gopkg.lock
-
4Gopkg.toml
-
6custom/conf/app.ini.sample
-
5docs/content/doc/advanced/config-cheat-sheet.en-us.md
-
1models/user.go
-
299modules/metrics/collector.go
-
11modules/setting/setting.go
-
30routers/metrics.go
-
10routers/routes/routes.go
-
20vendor/github.com/beorn7/perks/LICENSE
-
316vendor/github.com/beorn7/perks/quantile/stream.go
-
3vendor/github.com/golang/protobuf/LICENSE
-
56vendor/github.com/golang/protobuf/proto/clone.go
-
774vendor/github.com/golang/protobuf/proto/decode.go
-
350vendor/github.com/golang/protobuf/proto/discard.go
-
1186vendor/github.com/golang/protobuf/proto/encode.go
-
64vendor/github.com/golang/protobuf/proto/equal.go
-
322vendor/github.com/golang/protobuf/proto/extensions.go
-
135vendor/github.com/golang/protobuf/proto/lib.go
-
104vendor/github.com/golang/protobuf/proto/message_set.go
-
592vendor/github.com/golang/protobuf/proto/pointer_reflect.go
-
364vendor/github.com/golang/protobuf/proto/pointer_unsafe.go
-
470vendor/github.com/golang/protobuf/proto/properties.go
-
2767vendor/github.com/golang/protobuf/proto/table_marshal.go
-
654vendor/github.com/golang/protobuf/proto/table_merge.go
-
2051vendor/github.com/golang/protobuf/proto/table_unmarshal.go
-
78vendor/github.com/golang/protobuf/proto/text.go
-
167vendor/github.com/golang/protobuf/proto/text_parser.go
-
201vendor/github.com/matttproud/golang_protobuf_extensions/LICENSE
-
1vendor/github.com/matttproud/golang_protobuf_extensions/NOTICE
-
75vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/decode.go
-
16vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/doc.go
-
46vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/encode.go
-
201vendor/github.com/prometheus/client_golang/LICENSE
-
23vendor/github.com/prometheus/client_golang/NOTICE
-
120vendor/github.com/prometheus/client_golang/prometheus/collector.go
-
277vendor/github.com/prometheus/client_golang/prometheus/counter.go
-
184vendor/github.com/prometheus/client_golang/prometheus/desc.go
-
201vendor/github.com/prometheus/client_golang/prometheus/doc.go
-
119vendor/github.com/prometheus/client_golang/prometheus/expvar_collector.go
-
42vendor/github.com/prometheus/client_golang/prometheus/fnv.go
-
286vendor/github.com/prometheus/client_golang/prometheus/gauge.go
-
301vendor/github.com/prometheus/client_golang/prometheus/go_collector.go
-
614vendor/github.com/prometheus/client_golang/prometheus/histogram.go
-
505vendor/github.com/prometheus/client_golang/prometheus/http.go
-
85vendor/github.com/prometheus/client_golang/prometheus/internal/metric.go
-
70vendor/github.com/prometheus/client_golang/prometheus/labels.go
-
174vendor/github.com/prometheus/client_golang/prometheus/metric.go
-
52vendor/github.com/prometheus/client_golang/prometheus/observer.go
-
204vendor/github.com/prometheus/client_golang/prometheus/process_collector.go
-
199vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator.go
-
181vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_1_8.go
-
44vendor/github.com/prometheus/client_golang/prometheus/promhttp/delegator_pre_1_8.go
-
311vendor/github.com/prometheus/client_golang/prometheus/promhttp/http.go
-
97vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client.go
-
144vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_client_1_8.go
-
447vendor/github.com/prometheus/client_golang/prometheus/promhttp/instrument_server.go
-
895vendor/github.com/prometheus/client_golang/prometheus/registry.go
-
626vendor/github.com/prometheus/client_golang/prometheus/summary.go
-
51vendor/github.com/prometheus/client_golang/prometheus/timer.go
-
42vendor/github.com/prometheus/client_golang/prometheus/untyped.go
-
162vendor/github.com/prometheus/client_golang/prometheus/value.go
-
472vendor/github.com/prometheus/client_golang/prometheus/vec.go
-
179vendor/github.com/prometheus/client_golang/prometheus/wrap.go
-
201vendor/github.com/prometheus/client_model/LICENSE
-
5vendor/github.com/prometheus/client_model/NOTICE
-
629vendor/github.com/prometheus/client_model/go/metrics.pb.go
-
201vendor/github.com/prometheus/client_model/ruby/LICENSE
-
201vendor/github.com/prometheus/common/LICENSE
-
5vendor/github.com/prometheus/common/NOTICE
-
429vendor/github.com/prometheus/common/expfmt/decode.go
-
88vendor/github.com/prometheus/common/expfmt/encode.go
-
38vendor/github.com/prometheus/common/expfmt/expfmt.go
-
36vendor/github.com/prometheus/common/expfmt/fuzz.go
-
468vendor/github.com/prometheus/common/expfmt/text_create.go
-
757vendor/github.com/prometheus/common/expfmt/text_parse.go
-
162vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg.go
-
136vendor/github.com/prometheus/common/model/alert.go
-
105vendor/github.com/prometheus/common/model/fingerprinting.go
-
42vendor/github.com/prometheus/common/model/fnv.go
-
210vendor/github.com/prometheus/common/model/labels.go
-
169vendor/github.com/prometheus/common/model/labelset.go
-
103vendor/github.com/prometheus/common/model/metric.go
-
16vendor/github.com/prometheus/common/model/model.go
-
144vendor/github.com/prometheus/common/model/signature.go
-
106vendor/github.com/prometheus/common/model/silence.go
-
264vendor/github.com/prometheus/common/model/time.go
-
416vendor/github.com/prometheus/common/model/value.go
-
201vendor/github.com/prometheus/procfs/LICENSE
-
7vendor/github.com/prometheus/procfs/NOTICE
-
95vendor/github.com/prometheus/procfs/buddyinfo.go
-
45vendor/github.com/prometheus/procfs/doc.go
-
82vendor/github.com/prometheus/procfs/fs.go
-
59vendor/github.com/prometheus/procfs/internal/util/parse.go
-
45vendor/github.com/prometheus/procfs/internal/util/sysreadfile_linux.go
-
259vendor/github.com/prometheus/procfs/ipvs.go
-
151vendor/github.com/prometheus/procfs/mdstat.go
-
606vendor/github.com/prometheus/procfs/mountstats.go
-
216vendor/github.com/prometheus/procfs/net_dev.go
-
263vendor/github.com/prometheus/procfs/nfs/nfs.go
@ -0,0 +1,299 @@ |
|||
// Copyright 2018 The Gitea Authors. All rights reserved.
|
|||
// Use of this source code is governed by a MIT-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package metrics |
|||
|
|||
import ( |
|||
"code.gitea.io/gitea/models" |
|||
|
|||
"github.com/prometheus/client_golang/prometheus" |
|||
) |
|||
|
|||
const namespace = "gitea_" |
|||
|
|||
// Collector implements the prometheus.Collector interface and
|
|||
// exposes gitea metrics for prometheus
|
|||
type Collector struct { |
|||
Accesses *prometheus.Desc |
|||
Actions *prometheus.Desc |
|||
Attachments *prometheus.Desc |
|||
Comments *prometheus.Desc |
|||
Follows *prometheus.Desc |
|||
HookTasks *prometheus.Desc |
|||
Issues *prometheus.Desc |
|||
Labels *prometheus.Desc |
|||
LoginSources *prometheus.Desc |
|||
Milestones *prometheus.Desc |
|||
Mirrors *prometheus.Desc |
|||
Oauths *prometheus.Desc |
|||
Organizations *prometheus.Desc |
|||
PublicKeys *prometheus.Desc |
|||
Releases *prometheus.Desc |
|||
Repositories *prometheus.Desc |
|||
Stars *prometheus.Desc |
|||
Teams *prometheus.Desc |
|||
UpdateTasks *prometheus.Desc |
|||
Users *prometheus.Desc |
|||
Watches *prometheus.Desc |
|||
Webhooks *prometheus.Desc |
|||
} |
|||
|
|||
// NewCollector returns a new Collector with all prometheus.Desc initialized
|
|||
func NewCollector() Collector { |
|||
return Collector{ |
|||
Accesses: prometheus.NewDesc( |
|||
namespace+"accesses", |
|||
"Number of Accesses", |
|||
nil, nil, |
|||
), |
|||
Actions: prometheus.NewDesc( |
|||
namespace+"actions", |
|||
"Number of Actions", |
|||
nil, nil, |
|||
), |
|||
Attachments: prometheus.NewDesc( |
|||
namespace+"attachments", |
|||
"Number of Attachments", |
|||
nil, nil, |
|||
), |
|||
Comments: prometheus.NewDesc( |
|||
namespace+"comments", |
|||
"Number of Comments", |
|||
nil, nil, |
|||
), |
|||
Follows: prometheus.NewDesc( |
|||
namespace+"follows", |
|||
"Number of Follows", |
|||
nil, nil, |
|||
), |
|||
HookTasks: prometheus.NewDesc( |
|||
namespace+"hooktasks", |
|||
"Number of HookTasks", |
|||
nil, nil, |
|||
), |
|||
Issues: prometheus.NewDesc( |
|||
namespace+"issues", |
|||
"Number of Issues", |
|||
nil, nil, |
|||
), |
|||
Labels: prometheus.NewDesc( |
|||
namespace+"labels", |
|||
"Number of Labels", |
|||
nil, nil, |
|||
), |
|||
LoginSources: prometheus.NewDesc( |
|||
namespace+"loginsources", |
|||
"Number of LoginSources", |
|||
nil, nil, |
|||
), |
|||
Milestones: prometheus.NewDesc( |
|||
namespace+"milestones", |
|||
"Number of Milestones", |
|||
nil, nil, |
|||
), |
|||
Mirrors: prometheus.NewDesc( |
|||
namespace+"mirrors", |
|||
"Number of Mirrors", |
|||
nil, nil, |
|||
), |
|||
Oauths: prometheus.NewDesc( |
|||
namespace+"oauths", |
|||
"Number of Oauths", |
|||
nil, nil, |
|||
), |
|||
Organizations: prometheus.NewDesc( |
|||
namespace+"organizations", |
|||
"Number of Organizations", |
|||
nil, nil, |
|||
), |
|||
PublicKeys: prometheus.NewDesc( |
|||
namespace+"publickeys", |
|||
"Number of PublicKeys", |
|||
nil, nil, |
|||
), |
|||
Releases: prometheus.NewDesc( |
|||
namespace+"releases", |
|||
"Number of Releases", |
|||
nil, nil, |
|||
), |
|||
Repositories: prometheus.NewDesc( |
|||
namespace+"repositories", |
|||
"Number of Repositories", |
|||
nil, nil, |
|||
), |
|||
Stars: prometheus.NewDesc( |
|||
namespace+"stars", |
|||
"Number of Stars", |
|||
nil, nil, |
|||
), |
|||
Teams: prometheus.NewDesc( |
|||
namespace+"teams", |
|||
"Number of Teams", |
|||
nil, nil, |
|||
), |
|||
UpdateTasks: prometheus.NewDesc( |
|||
namespace+"updatetasks", |
|||
"Number of UpdateTasks", |
|||
nil, nil, |
|||
), |
|||
Users: prometheus.NewDesc( |
|||
namespace+"users", |
|||
"Number of Users", |
|||
nil, nil, |
|||
), |
|||
Watches: prometheus.NewDesc( |
|||
namespace+"watches", |
|||
"Number of Watches", |
|||
nil, nil, |
|||
), |
|||
Webhooks: prometheus.NewDesc( |
|||
namespace+"webhooks", |
|||
"Number of Webhooks", |
|||
nil, nil, |
|||
), |
|||
} |
|||
|
|||
} |
|||
|
|||
// Describe returns all possible prometheus.Desc
|
|||
func (c Collector) Describe(ch chan<- *prometheus.Desc) { |
|||
ch <- c.Accesses |
|||
ch <- c.Actions |
|||
ch <- c.Attachments |
|||
ch <- c.Comments |
|||
ch <- c.Follows |
|||
ch <- c.HookTasks |
|||
ch <- c.Issues |
|||
ch <- c.Labels |
|||
ch <- c.LoginSources |
|||
ch <- c.Milestones |
|||
ch <- c.Mirrors |
|||
ch <- c.Oauths |
|||
ch <- c.Organizations |
|||
ch <- c.PublicKeys |
|||
ch <- c.Releases |
|||
ch <- c.Repositories |
|||
ch <- c.Stars |
|||
ch <- c.Teams |
|||
ch <- c.UpdateTasks |
|||
ch <- c.Users |
|||
ch <- c.Watches |
|||
ch <- c.Webhooks |
|||
} |
|||
|
|||
// Collect returns the metrics with values
|
|||
func (c Collector) Collect(ch chan<- prometheus.Metric) { |
|||
stats := models.GetStatistic() |
|||
|
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Accesses, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Access), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Actions, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Action), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Attachments, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Attachment), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Comments, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Comment), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Follows, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Follow), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.HookTasks, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.HookTask), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Issues, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Issue), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Labels, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Label), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.LoginSources, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.LoginSource), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Milestones, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Milestone), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Mirrors, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Mirror), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Oauths, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Oauth), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Organizations, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Org), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.PublicKeys, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.PublicKey), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Releases, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Release), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Repositories, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Repo), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Stars, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Star), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Teams, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Team), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.UpdateTasks, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.UpdateTask), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Users, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.User), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Watches, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Watch), |
|||
) |
|||
ch <- prometheus.MustNewConstMetric( |
|||
c.Webhooks, |
|||
prometheus.GaugeValue, |
|||
float64(stats.Counter.Webhook), |
|||
) |
|||
} |
@ -0,0 +1,30 @@ |
|||
// Copyright 2018 The Gitea Authors. All rights reserved.
|
|||
// Use of this source code is governed by a MIT-style
|
|||
// license that can be found in the LICENSE file.
|
|||
|
|||
package routers |
|||
|
|||
import ( |
|||
"github.com/prometheus/client_golang/prometheus/promhttp" |
|||
|
|||
"code.gitea.io/gitea/modules/context" |
|||
"code.gitea.io/gitea/modules/setting" |
|||
) |
|||
|
|||
// Metrics validate auth token and render prometheus metrics
|
|||
func Metrics(ctx *context.Context) { |
|||
if setting.Metrics.Token == "" { |
|||
promhttp.Handler().ServeHTTP(ctx.Resp, ctx.Req.Request) |
|||
return |
|||
} |
|||
header := ctx.Header().Get("Authorization") |
|||
if header == "" { |
|||
ctx.Error(401) |
|||
return |
|||
} |
|||
if header != "Bearer "+setting.Metrics.Token { |
|||
ctx.Error(401) |
|||
return |
|||
} |
|||
promhttp.Handler().ServeHTTP(ctx.Resp, ctx.Req.Request) |
|||
} |
@ -0,0 +1,20 @@ |
|||
Copyright (C) 2013 Blake Mizerany |
|||
|
|||
Permission is hereby granted, free of charge, to any person obtaining |
|||
a copy of this software and associated documentation files (the |
|||
"Software"), to deal in the Software without restriction, including |
|||
without limitation the rights to use, copy, modify, merge, publish, |
|||
distribute, sublicense, and/or sell copies of the Software, and to |
|||
permit persons to whom the Software is furnished to do so, subject to |
|||
the following conditions: |
|||
|
|||
The above copyright notice and this permission notice shall be |
|||
included in all copies or substantial portions of the Software. |
|||
|
|||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
|||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
|||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
|||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
|||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
|||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
|||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
@ -0,0 +1,316 @@ |
|||
// Package quantile computes approximate quantiles over an unbounded data
|
|||
// stream within low memory and CPU bounds.
|
|||
//
|
|||
// A small amount of accuracy is traded to achieve the above properties.
|
|||
//
|
|||
// Multiple streams can be merged before calling Query to generate a single set
|
|||
// of results. This is meaningful when the streams represent the same type of
|
|||
// data. See Merge and Samples.
|
|||
//
|
|||
// For more detailed information about the algorithm used, see:
|
|||
//
|
|||
// Effective Computation of Biased Quantiles over Data Streams
|
|||
//
|
|||
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
|
|||
package quantile |
|||
|
|||
import ( |
|||
"math" |
|||
"sort" |
|||
) |
|||
|
|||
// Sample holds an observed value and meta information for compression. JSON
|
|||
// tags have been added for convenience.
|
|||
type Sample struct { |
|||
Value float64 `json:",string"` |
|||
Width float64 `json:",string"` |
|||
Delta float64 `json:",string"` |
|||
} |
|||
|
|||
// Samples represents a slice of samples. It implements sort.Interface.
|
|||
type Samples []Sample |
|||
|
|||
func (a Samples) Len() int { return len(a) } |
|||
func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value } |
|||
func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] } |
|||
|
|||
type invariant func(s *stream, r float64) float64 |
|||
|
|||
// NewLowBiased returns an initialized Stream for low-biased quantiles
|
|||
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
|
|||
// error guarantees can still be given even for the lower ranks of the data
|
|||
// distribution.
|
|||
//
|
|||
// The provided epsilon is a relative error, i.e. the true quantile of a value
|
|||
// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
|
|||
//
|
|||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
|
|||
// properties.
|
|||
func NewLowBiased(epsilon float64) *Stream { |
|||
ƒ := func(s *stream, r float64) float64 { |
|||
return 2 * epsilon * r |
|||
} |
|||
return newStream(ƒ) |
|||
} |
|||
|
|||
// NewHighBiased returns an initialized Stream for high-biased quantiles
|
|||
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
|
|||
// error guarantees can still be given even for the higher ranks of the data
|
|||
// distribution.
|
|||
//
|
|||
// The provided epsilon is a relative error, i.e. the true quantile of a value
|
|||
// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
|
|||
//
|
|||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
|
|||
// properties.
|
|||
func NewHighBiased(epsilon float64) *Stream { |
|||
ƒ := func(s *stream, r float64) float64 { |
|||
return 2 * epsilon * (s.n - r) |
|||
} |
|||
return newStream(ƒ) |
|||
} |
|||
|
|||
// NewTargeted returns an initialized Stream concerned with a particular set of
|
|||
// quantile values that are supplied a priori. Knowing these a priori reduces
|
|||
// space and computation time. The targets map maps the desired quantiles to
|
|||
// their absolute errors, i.e. the true quantile of a value returned by a query
|
|||
// is guaranteed to be within (Quantile±Epsilon).
|
|||
//
|
|||
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
|
|||
func NewTargeted(targetMap map[float64]float64) *Stream { |
|||
// Convert map to slice to avoid slow iterations on a map.
|
|||
// ƒ is called on the hot path, so converting the map to a slice
|
|||
// beforehand results in significant CPU savings.
|
|||
targets := targetMapToSlice(targetMap) |
|||
|
|||
ƒ := func(s *stream, r float64) float64 { |
|||
var m = math.MaxFloat64 |
|||
var f float64 |
|||
for _, t := range targets { |
|||
if t.quantile*s.n <= r { |
|||
f = (2 * t.epsilon * r) / t.quantile |
|||
} else { |
|||
f = (2 * t.epsilon * (s.n - r)) / (1 - t.quantile) |
|||
} |
|||
if f < m { |
|||
m = f |
|||
} |
|||
} |
|||
return m |
|||
} |
|||
return newStream(ƒ) |
|||
} |
|||
|
|||
type target struct { |
|||
quantile float64 |
|||
epsilon float64 |
|||
} |
|||
|
|||
func targetMapToSlice(targetMap map[float64]float64) []target { |
|||
targets := make([]target, 0, len(targetMap)) |
|||
|
|||
for quantile, epsilon := range targetMap { |
|||
t := target{ |
|||
quantile: quantile, |
|||
epsilon: epsilon, |
|||
} |
|||
targets = append(targets, t) |
|||
} |
|||
|
|||
return targets |
|||
} |
|||
|
|||
// Stream computes quantiles for a stream of float64s. It is not thread-safe by
|
|||
// design. Take care when using across multiple goroutines.
|
|||
type Stream struct { |
|||
*stream |
|||
b Samples |
|||
sorted bool |
|||
} |
|||
|
|||
func newStream(ƒ invariant) *Stream { |
|||
x := &stream{ƒ: ƒ} |
|||
return &Stream{x, make(Samples, 0, 500), true} |
|||
} |
|||
|
|||
// Insert inserts v into the stream.
|
|||
func (s *Stream) Insert(v float64) { |
|||
s.insert(Sample{Value: v, Width: 1}) |
|||
} |
|||
|
|||
func (s *Stream) insert(sample Sample) { |
|||
s.b = append(s.b, sample) |
|||
s.sorted = false |
|||
if len(s.b) == cap(s.b) { |
|||
s.flush() |
|||
} |
|||
} |
|||
|
|||
// Query returns the computed qth percentiles value. If s was created with
|
|||
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
|
|||
// will return an unspecified result.
|
|||
func (s *Stream) Query(q float64) float64 { |
|||
if !s.flushed() { |
|||
// Fast path when there hasn't been enough data for a flush;
|
|||
// this also yields better accuracy for small sets of data.
|
|||
l := len(s.b) |
|||
if l == 0 { |
|||
return 0 |
|||
} |
|||
i := int(math.Ceil(float64(l) * q)) |
|||
if i > 0 { |
|||
i -= 1 |
|||
} |
|||
s.maybeSort() |
|||
return s.b[i].Value |
|||
} |
|||
s.flush() |
|||
return s.stream.query(q) |
|||
} |
|||
|
|||
// Merge merges samples into the underlying streams samples. This is handy when
|
|||
// merging multiple streams from separate threads, database shards, etc.
|
|||
//
|
|||
// ATTENTION: This method is broken and does not yield correct results. The
|
|||
// underlying algorithm is not capable of merging streams correctly.
|
|||
func (s *Stream) Merge(samples Samples) { |
|||
sort.Sort(samples) |
|||
s.stream.merge(samples) |
|||
} |
|||
|
|||
// Reset reinitializes and clears the list reusing the samples buffer memory.
|
|||
func (s *Stream) Reset() { |
|||
s.stream.reset() |
|||
s.b = s.b[:0] |
|||
} |
|||
|
|||
// Samples returns stream samples held by s.
|
|||
func (s *Stream) Samples() Samples { |
|||
if !s.flushed() { |
|||
return s.b |
|||
} |
|||
s.flush() |
|||
return s.stream.samples() |
|||
} |
|||
|
|||
// Count returns the total number of samples observed in the stream
|
|||
// since initialization.
|
|||
func (s *Stream) Count() int { |
|||
return len(s.b) + s.stream.count() |
|||
} |
|||
|
|||
func (s *Stream) flush() { |
|||
s.maybeSort() |
|||
s.stream.merge(s.b) |
|||
s.b = s.b[:0] |
|||
} |
|||
|
|||
func (s *Stream) maybeSort() { |
|||
if !s.sorted { |
|||
s.sorted = true |
|||
sort.Sort(s.b) |
|||
} |
|||
} |
|||
|
|||
func (s *Stream) flushed() bool { |
|||
return len(s.stream.l) > 0 |
|||
} |
|||
|
|||
type stream struct { |
|||
n float64 |
|||
l []Sample |
|||
ƒ invariant |
|||
} |
|||
|
|||
func (s *stream) reset() { |
|||
s.l = s.l[:0] |
|||
s.n = 0 |
|||
} |
|||
|
|||
func (s *stream) insert(v float64) { |
|||
s.merge(Samples{{v, 1, 0}}) |
|||
} |
|||
|
|||
func (s *stream) merge(samples Samples) { |
|||
// TODO(beorn7): This tries to merge not only individual samples, but
|
|||
// whole summaries. The paper doesn't mention merging summaries at
|
|||
// all. Unittests show that the merging is inaccurate. Find out how to
|
|||
// do merges properly.
|
|||
var r float64 |
|||
i := 0 |
|||
for _, sample := range samples { |
|||
for ; i < len(s.l); i++ { |
|||
c := s.l[i] |
|||
if c.Value > sample.Value { |
|||
// Insert at position i.
|
|||
s.l = append(s.l, Sample{}) |
|||
copy(s.l[i+1:], s.l[i:]) |
|||
s.l[i] = Sample{ |
|||
sample.Value, |
|||
sample.Width, |
|||
math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1), |
|||
// TODO(beorn7): How to calculate delta correctly?
|
|||
} |
|||
i++ |
|||
goto inserted |
|||
} |
|||
r += c.Width |
|||
} |
|||
s.l = append(s.l, Sample{sample.Value, sample.Width, 0}) |
|||
i++ |
|||
inserted: |
|||
s.n += sample.Width |
|||
r += sample.Width |
|||
} |
|||
s.compress() |
|||
} |
|||
|
|||
func (s *stream) count() int { |
|||
return int(s.n) |
|||
} |
|||
|
|||
func (s *stream) query(q float64) float64 { |
|||
t := math.Ceil(q * s.n) |
|||
t += math.Ceil(s.ƒ(s, t) / 2) |
|||
p := s.l[0] |
|||
var r float64 |
|||
for _, c := range s.l[1:] { |
|||
r += p.Width |
|||
if r+c.Width+c.Delta > t { |
|||
return p.Value |
|||
} |
|||
p = c |
|||
} |
|||
return p.Value |
|||
} |
|||
|
|||
func (s *stream) compress() { |
|||
if len(s.l) < 2 { |
|||
return |
|||
} |
|||
x := s.l[len(s.l)-1] |
|||
xi := len(s.l) - 1 |
|||
r := s.n - 1 - x.Width |
|||
|
|||
for i := len(s.l) - 2; i >= 0; i-- { |
|||
c := s.l[i] |
|||
if c.Width+x.Width+x.Delta <= s.ƒ(s, r) { |
|||
x.Width += c.Width |
|||
s.l[xi] = x |
|||
// Remove element at i.
|
|||
copy(s.l[i:], s.l[i+1:]) |
|||
s.l = s.l[:len(s.l)-1] |
|||
xi -= 1 |
|||
} else { |
|||
x = c |
|||
xi = i |
|||
} |
|||
r -= c.Width |
|||
} |
|||
} |
|||
|
|||
func (s *stream) samples() Samples { |
|||
samples := make(Samples, len(s.l)) |
|||
copy(samples, s.l) |
|||
return samples |
|||
} |