Prometheus endpoint (#5256)

* Add prometheus collector and route

* dep ensure -add github.com/prometheus/client_golang/prometheus

* dep ensure -update github.com/golang/protobuf

* add metrics to reserved usernames

* add comment head in metrics package

* fix style imports

* add metrics settings

* add bearer token check

* mapping metrics configs

* fix lint

* update config cheat sheet

* update conf sample, typo fix
release/v1.7
Stanislav 5 years ago committed by techknowlogick
parent 92fb89f6e1
commit 078c404c3b

68
Gopkg.lock generated

@ -81,6 +81,14 @@
pruneopts = "NUT"
revision = "349dd0209470eabd9514242c688c403c0926d266"
[[projects]]
branch = "master"
digest = "1:707ebe952a8b3d00b343c01536c79c73771d100f63ec6babeaed5c79e2b8a8dd"
name = "github.com/beorn7/perks"
packages = ["quantile"]
pruneopts = "NUT"
revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
[[projects]]
digest = "1:67351095005f164e748a5a21899d1403b03878cb2d40a7b0f742376e6eeda974"
name = "github.com/blevesearch/bleve"
@ -405,11 +413,12 @@
revision = "7f3990acf1833faa5ebd0e86f0a4c72a4b5eba3c"
[[projects]]
digest = "1:b64f9be717fdab5f75122dc3868e8ca9d003779b6bc55f64f39a0cddc698bf88"
digest = "1:97df918963298c287643883209a2c3f642e6593379f97ab400c2a2e219ab647d"
name = "github.com/golang/protobuf"
packages = ["proto"]
pruneopts = "NUT"
revision = "99511271042a09d1e01baea8781caa5210fec66e"
revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
version = "v1.2.0"
[[projects]]
digest = "1:60e25fc5f5cfd7783f985ca99b4383e848981dddf0be584db7d809be20848e25"
@ -573,6 +582,14 @@
pruneopts = "NUT"
revision = "c7c4067b79cc51e6dfdcef5c702e74b1e0fa7c75"
[[projects]]
digest = "1:5985ef4caf91ece5d54817c11ea25f182697534f8ae6521eadcd628c142ac4b6"
name = "github.com/matttproud/golang_protobuf_extensions"
packages = ["pbutil"]
pruneopts = "NUT"
revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
version = "v1.0.1"
[[projects]]
branch = "master"
digest = "1:1ed0f3c066eb9d1c2ff7a864a6fa595c70b9b49049cc46af6a6f7ff0e4655321"
@ -644,6 +661,51 @@
pruneopts = "NUT"
revision = "54653902c20e47f3417541d35435cb6d6162e28a"
[[projects]]
digest = "1:aa2da1df3327c3a338bb42f978407c07de74cd0a5bef35e9411881dffd444214"
name = "github.com/prometheus/client_golang"
packages = [
"prometheus",
"prometheus/internal",
"prometheus/promhttp",
]
pruneopts = "NUT"
revision = "1cafe34db7fdec6022e17e00e1c1ea501022f3e4"
version = "v0.9.0"
[[projects]]
branch = "master"
digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4"
name = "github.com/prometheus/client_model"
packages = ["go"]
pruneopts = "NUT"
revision = "5c3871d89910bfb32f5fcab2aa4b9ec68e65a99f"
[[projects]]
branch = "master"
digest = "1:06375f3b602de9c99fa99b8484f0e949fd5273e6e9c6592b5a0dd4cd9085f3ea"
name = "github.com/prometheus/common"
packages = [
"expfmt",
"internal/bitbucket.org/ww/goautoneg",
"model",
]
pruneopts = "NUT"
revision = "7e9e6cabbd393fc208072eedef99188d0ce788b6"
[[projects]]
branch = "master"
digest = "1:102dea0c03a915acfc634b7c67f2662012b5483b56d9025e33f5188e112759b6"
name = "github.com/prometheus/procfs"
packages = [
".",
"internal/util",
"nfs",
"xfs",
]
pruneopts = "NUT"
revision = "185b4288413d2a0dd0806f78c90dde719829e5ae"
[[projects]]
branch = "master"
digest = "1:5be01c22bc1040e2f6ce4755d51a0ac9cef823a9f2004fb1f9896a414ef519e6"
@ -981,6 +1043,8 @@
"github.com/nfnt/resize",
"github.com/pquerna/otp",
"github.com/pquerna/otp/totp",
"github.com/prometheus/client_golang/prometheus",
"github.com/prometheus/client_golang/prometheus/promhttp",
"github.com/russross/blackfriday",
"github.com/satori/go.uuid",
"github.com/sergi/go-diff/diffmatchpatch",

@ -107,3 +107,7 @@ ignored = ["google.golang.org/appengine*"]
[[override]]
revision = "c10ba270aa0bf8b8c1c986e103859c67a9103061"
name = "golang.org/x/oauth2"
[[constraint]]
name = "github.com/prometheus/client_golang"
version = "0.9.0"

@ -659,3 +659,9 @@ FILE_EXTENSIONS = .adoc,.asciidoc
RENDER_COMMAND = "asciidoc --out-file=- -"
; Don't pass the file on STDIN, pass the filename as argument instead.
IS_INPUT_FILE = false
[metrics]
; Enables metrics endpoint. True or false; default is false.
ENABLED = false
; If you want to add authorization, specify a token here
TOKEN =

@ -302,6 +302,11 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
- `PULL`: **300**: Git pull from internal repositories timeout seconds.
- `GC`: **60**: Git repository GC timeout seconds.
## Metrics (`metrics`)
- `ENABLED`: **false**: Enables /metrics endpoint for prometheus.
- `TOKEN`: **\<empty\>**: You need to specify the token, if you want to include in the authorization the metrics . The same token need to be used in prometheus parameters `bearer_token` or `bearer_token_file`.
## API (`api`)
- `ENABLE_SWAGGER_ENDPOINT`: **true**: Enables /api/swagger, /api/v1/swagger etc. endpoints. True or false; default is true.

@ -698,6 +698,7 @@ var (
"issues",
"js",
"less",
"metrics",
"new",
"org",
"plugins",

@ -0,0 +1,299 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package metrics
import (
"code.gitea.io/gitea/models"
"github.com/prometheus/client_golang/prometheus"
)
const namespace = "gitea_"
// Collector implements the prometheus.Collector interface and
// exposes gitea metrics for prometheus
type Collector struct {
Accesses *prometheus.Desc
Actions *prometheus.Desc
Attachments *prometheus.Desc
Comments *prometheus.Desc
Follows *prometheus.Desc
HookTasks *prometheus.Desc
Issues *prometheus.Desc
Labels *prometheus.Desc
LoginSources *prometheus.Desc
Milestones *prometheus.Desc
Mirrors *prometheus.Desc
Oauths *prometheus.Desc
Organizations *prometheus.Desc
PublicKeys *prometheus.Desc
Releases *prometheus.Desc
Repositories *prometheus.Desc
Stars *prometheus.Desc
Teams *prometheus.Desc
UpdateTasks *prometheus.Desc
Users *prometheus.Desc
Watches *prometheus.Desc
Webhooks *prometheus.Desc
}
// NewCollector returns a new Collector with all prometheus.Desc initialized
func NewCollector() Collector {
return Collector{
Accesses: prometheus.NewDesc(
namespace+"accesses",
"Number of Accesses",
nil, nil,
),
Actions: prometheus.NewDesc(
namespace+"actions",
"Number of Actions",
nil, nil,
),
Attachments: prometheus.NewDesc(
namespace+"attachments",
"Number of Attachments",
nil, nil,
),
Comments: prometheus.NewDesc(
namespace+"comments",
"Number of Comments",
nil, nil,
),
Follows: prometheus.NewDesc(
namespace+"follows",
"Number of Follows",
nil, nil,
),
HookTasks: prometheus.NewDesc(
namespace+"hooktasks",
"Number of HookTasks",
nil, nil,
),
Issues: prometheus.NewDesc(
namespace+"issues",
"Number of Issues",
nil, nil,
),
Labels: prometheus.NewDesc(
namespace+"labels",
"Number of Labels",
nil, nil,
),
LoginSources: prometheus.NewDesc(
namespace+"loginsources",
"Number of LoginSources",
nil, nil,
),
Milestones: prometheus.NewDesc(
namespace+"milestones",
"Number of Milestones",
nil, nil,
),
Mirrors: prometheus.NewDesc(
namespace+"mirrors",
"Number of Mirrors",
nil, nil,
),
Oauths: prometheus.NewDesc(
namespace+"oauths",
"Number of Oauths",
nil, nil,
),
Organizations: prometheus.NewDesc(
namespace+"organizations",
"Number of Organizations",
nil, nil,
),
PublicKeys: prometheus.NewDesc(
namespace+"publickeys",
"Number of PublicKeys",
nil, nil,
),
Releases: prometheus.NewDesc(
namespace+"releases",
"Number of Releases",
nil, nil,
),
Repositories: prometheus.NewDesc(
namespace+"repositories",
"Number of Repositories",
nil, nil,
),
Stars: prometheus.NewDesc(
namespace+"stars",
"Number of Stars",
nil, nil,
),
Teams: prometheus.NewDesc(
namespace+"teams",
"Number of Teams",
nil, nil,
),
UpdateTasks: prometheus.NewDesc(
namespace+"updatetasks",
"Number of UpdateTasks",
nil, nil,
),
Users: prometheus.NewDesc(
namespace+"users",
"Number of Users",
nil, nil,
),
Watches: prometheus.NewDesc(
namespace+"watches",
"Number of Watches",
nil, nil,
),
Webhooks: prometheus.NewDesc(
namespace+"webhooks",
"Number of Webhooks",
nil, nil,
),
}
}
// Describe returns all possible prometheus.Desc
func (c Collector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.Accesses
ch <- c.Actions
ch <- c.Attachments
ch <- c.Comments
ch <- c.Follows
ch <- c.HookTasks
ch <- c.Issues
ch <- c.Labels
ch <- c.LoginSources
ch <- c.Milestones
ch <- c.Mirrors
ch <- c.Oauths
ch <- c.Organizations
ch <- c.PublicKeys
ch <- c.Releases
ch <- c.Repositories
ch <- c.Stars
ch <- c.Teams
ch <- c.UpdateTasks
ch <- c.Users
ch <- c.Watches
ch <- c.Webhooks
}
// Collect returns the metrics with values
func (c Collector) Collect(ch chan<- prometheus.Metric) {
stats := models.GetStatistic()
ch <- prometheus.MustNewConstMetric(
c.Accesses,
prometheus.GaugeValue,
float64(stats.Counter.Access),
)
ch <- prometheus.MustNewConstMetric(
c.Actions,
prometheus.GaugeValue,
float64(stats.Counter.Action),
)
ch <- prometheus.MustNewConstMetric(
c.Attachments,
prometheus.GaugeValue,
float64(stats.Counter.Attachment),
)
ch <- prometheus.MustNewConstMetric(
c.Comments,
prometheus.GaugeValue,
float64(stats.Counter.Comment),
)
ch <- prometheus.MustNewConstMetric(
c.Follows,
prometheus.GaugeValue,
float64(stats.Counter.Follow),
)
ch <- prometheus.MustNewConstMetric(
c.HookTasks,
prometheus.GaugeValue,
float64(stats.Counter.HookTask),
)
ch <- prometheus.MustNewConstMetric(
c.Issues,
prometheus.GaugeValue,
float64(stats.Counter.Issue),
)
ch <- prometheus.MustNewConstMetric(
c.Labels,
prometheus.GaugeValue,
float64(stats.Counter.Label),
)
ch <- prometheus.MustNewConstMetric(
c.LoginSources,
prometheus.GaugeValue,
float64(stats.Counter.LoginSource),
)
ch <- prometheus.MustNewConstMetric(
c.Milestones,
prometheus.GaugeValue,
float64(stats.Counter.Milestone),
)
ch <- prometheus.MustNewConstMetric(
c.Mirrors,
prometheus.GaugeValue,
float64(stats.Counter.Mirror),
)
ch <- prometheus.MustNewConstMetric(
c.Oauths,
prometheus.GaugeValue,
float64(stats.Counter.Oauth),
)
ch <- prometheus.MustNewConstMetric(
c.Organizations,
prometheus.GaugeValue,
float64(stats.Counter.Org),
)
ch <- prometheus.MustNewConstMetric(
c.PublicKeys,
prometheus.GaugeValue,
float64(stats.Counter.PublicKey),
)
ch <- prometheus.MustNewConstMetric(
c.Releases,
prometheus.GaugeValue,
float64(stats.Counter.Release),
)
ch <- prometheus.MustNewConstMetric(
c.Repositories,
prometheus.GaugeValue,
float64(stats.Counter.Repo),
)
ch <- prometheus.MustNewConstMetric(
c.Stars,
prometheus.GaugeValue,
float64(stats.Counter.Star),
)
ch <- prometheus.MustNewConstMetric(
c.Teams,
prometheus.GaugeValue,
float64(stats.Counter.Team),
)
ch <- prometheus.MustNewConstMetric(
c.UpdateTasks,
prometheus.GaugeValue,
float64(stats.Counter.UpdateTask),
)
ch <- prometheus.MustNewConstMetric(
c.Users,
prometheus.GaugeValue,
float64(stats.Counter.User),
)
ch <- prometheus.MustNewConstMetric(
c.Watches,
prometheus.GaugeValue,
float64(stats.Counter.Watch),
)
ch <- prometheus.MustNewConstMetric(
c.Webhooks,
prometheus.GaugeValue,
float64(stats.Counter.Webhook),
)
}

@ -561,6 +561,15 @@ var (
TrustedFacets []string
}{}
// Metrics settings
Metrics = struct {
Enabled bool
Token string
}{
Enabled: false,
Token: "",
}
// I18n settings
Langs []string
Names []string
@ -1125,6 +1134,8 @@ func NewContext() {
log.Fatal(4, "Failed to map Git settings: %v", err)
} else if err = Cfg.Section("api").MapTo(&API); err != nil {
log.Fatal(4, "Failed to map API settings: %v", err)
} else if err = Cfg.Section("metrics").MapTo(&Metrics); err != nil {
log.Fatal(4, "Failed to map Metrics settings: %v", err)
}
sec = Cfg.Section("mirror")

@ -0,0 +1,30 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package routers
import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/setting"
)
// Metrics validate auth token and render prometheus metrics
func Metrics(ctx *context.Context) {
if setting.Metrics.Token == "" {
promhttp.Handler().ServeHTTP(ctx.Resp, ctx.Req.Request)
return
}
header := ctx.Header().Get("Authorization")
if header == "" {
ctx.Error(401)
return
}
if header != "Bearer "+setting.Metrics.Token {
ctx.Error(401)
return
}
promhttp.Handler().ServeHTTP(ctx.Resp, ctx.Req.Request)
}

@ -16,6 +16,7 @@ import (
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/metrics"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/public"
"code.gitea.io/gitea/modules/setting"
@ -39,6 +40,7 @@ import (
"github.com/go-macaron/i18n"
"github.com/go-macaron/session"
"github.com/go-macaron/toolbox"
"github.com/prometheus/client_golang/prometheus"
"github.com/tstranex/u2f"
"gopkg.in/macaron.v1"
)
@ -788,6 +790,14 @@ func RegisterRoutes(m *macaron.Macaron) {
}
})
// prometheus metrics endpoint
if setting.Metrics.Enabled {
c := metrics.NewCollector()
prometheus.MustRegister(c)
m.Get("/metrics", routers.Metrics)
}
// Not found handler.
m.NotFound(routers.NotFound)
}

@ -0,0 +1,20 @@
Copyright (C) 2013 Blake Mizerany
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -0,0 +1,316 @@
// Package quantile computes approximate quantiles over an unbounded data
// stream within low memory and CPU bounds.
//
// A small amount of accuracy is traded to achieve the above properties.
//
// Multiple streams can be merged before calling Query to generate a single set
// of results. This is meaningful when the streams represent the same type of
// data. See Merge and Samples.
//
// For more detailed information about the algorithm used, see:
//
// Effective Computation of Biased Quantiles over Data Streams
//
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
package quantile
import (
"math"
"sort"
)
// Sample holds an observed value and meta information for compression. JSON
// tags have been added for convenience.
type Sample struct {
Value float64 `json:",string"`
Width float64 `json:",string"`
Delta float64 `json:",string"`
}
// Samples represents a slice of samples. It implements sort.Interface.
type Samples []Sample
func (a Samples) Len() int { return len(a) }
func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
type invariant func(s *stream, r float64) float64
// NewLowBiased returns an initialized Stream for low-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the lower ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewLowBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * r
}
return newStream(ƒ)
}
// NewHighBiased returns an initialized Stream for high-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the higher ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewHighBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * (s.n - r)
}
return newStream(ƒ)
}
// NewTargeted returns an initialized Stream concerned with a particular set of
// quantile values that are supplied a priori. Knowing these a priori reduces
// space and computation time. The targets map maps the desired quantiles to
// their absolute errors, i.e. the true quantile of a value returned by a query
// is guaranteed to be within (Quantile±Epsilon).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewTargeted(targetMap map[float64]float64) *Stream {
// Convert map to slice to avoid slow iterations on a map.
// ƒ is called on the hot path, so converting the map to a slice
// beforehand results in significant CPU savings.
targets := targetMapToSlice(targetMap)
ƒ := func(s *stream, r float64) float64 {
var m = math.MaxFloat64
var f float64
for _, t := range targets {
if t.quantile*s.n <= r {
f = (2 * t.epsilon * r) / t.quantile
} else {
f = (2 * t.epsilon * (s.n - r)) / (1 - t.quantile)
}
if f < m {
m = f
}
}
return m
}
return newStream(ƒ)
}
type target struct {
quantile float64
epsilon float64
}
func targetMapToSlice(targetMap map[float64]float64) []target {
targets := make([]target, 0, len(targetMap))
for quantile, epsilon := range targetMap {
t := target{
quantile: quantile,
epsilon: epsilon,
}
targets = append(targets, t)
}
return targets
}
// Stream computes quantiles for a stream of float64s. It is not thread-safe by
// design. Take care when using across multiple goroutines.
type Stream struct {
*stream
b Samples
sorted bool
}
func newStream(ƒ invariant) *Stream {
x := &stream{ƒ: ƒ}
return &Stream{x, make(Samples, 0, 500), true}
}
// Insert inserts v into the stream.
func (s *Stream) Insert(v float64) {
s.insert(Sample{Value: v, Width: 1})
}
func (s *Stream) insert(sample Sample) {
s.b = append(s.b, sample)
s.sorted = false
if len(s.b) == cap(s.b) {
s.flush()
}
}
// Query returns the computed qth percentiles value. If s was created with
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
// will return an unspecified result.
func (s *Stream) Query(q float64) float64 {
if !s.flushed() {
// Fast path when there hasn't been enough data for a flush;
// this also yields better accuracy for small sets of data.
l := len(s.b)
if l == 0 {
return 0
}
i := int(math.Ceil(float64(l) * q))
if i > 0 {
i -= 1
}
s.maybeSort()
return s.b[i].Value
}
s.flush()
return s.stream.query(q)
}
// Merge merges samples into the underlying streams samples. This is handy when
// merging multiple streams from separate threads, database shards, etc.
//
// ATTENTION: This method is broken and does not yield correct results. The
// underlying algorithm is not capable of merging streams correctly.
func (s *Stream) Merge(samples Samples) {
sort.Sort(samples)
s.stream.merge(samples)
}
// Reset reinitializes and clears the list reusing the samples buffer memory.
func (s *Stream) Reset() {
s.stream.reset()
s.b = s.b[:0]
}
// Samples returns stream samples held by s.
func (s *Stream) Samples() Samples {
if !s.flushed() {
return s.b
}
s.flush()
return s.stream.samples()
}
// Count returns the total number of samples observed in the stream
// since initialization.
func (s *Stream) Count() int {
return len(s.b) + s.stream.count()
}
func (s *Stream) flush() {
s.maybeSort()
s.stream.merge(s.b)
s.b = s.b[:0]
}
func (s *Stream) maybeSort() {
if !s.sorted {
s.sorted = true
sort.Sort(s.b)
}
}
func (s *Stream) flushed() bool {
return len(s.stream.l) > 0
}
type stream struct {
n float64
l []Sample
ƒ invariant
}
func (s *stream) reset() {
s.l = s.l[:0]
s.n = 0
}
func (s *stream) insert(v float64) {
s.merge(Samples{{v, 1, 0}})
}
func (s *stream) merge(samples Samples) {
// TODO(beorn7): This tries to merge not only individual samples, but
// whole summaries. The paper doesn't mention merging summaries at
// all. Unittests show that the merging is inaccurate. Find out how to
// do merges properly.
var r float64
i := 0
for _, sample := range samples {
for ; i < len(s.l); i++ {
c := s.l[i]
if c.Value > sample.Value {
// Insert at position i.
s.l = append(s.l, Sample{})
copy(s.l[i+1:], s.l[i:])
s.l[i] = Sample{
sample.Value,
sample.Width,
math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
// TODO(beorn7): How to calculate delta correctly?
}
i++
goto inserted
}
r += c.Width
}
s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
i++
inserted:
s.n += sample.Width
r += sample.Width
}
s.compress()
}
func (s *stream) count() int {
return int(s.n)
}
func (s *stream) query(q float64) float64 {
t := math.Ceil(q * s.n)
t += math.Ceil(s.ƒ(s, t) / 2)
p := s.l[0]
var r float64
for _, c := range s.l[1:] {
r += p.Width
if r+c.Width+c.Delta > t {
return p.Value
}
p = c
}
return p.Value
}
func (s *stream) compress() {
if len(s.l) < 2 {
return
}
x := s.l[len(s.l)-1]
xi := len(s.l) - 1
r := s.n - 1 - x.Width
for i := len(s.l) - 2; i >= 0; i-- {
c := s.l[i]
if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
x.Width += c.Width
s.l[xi] = x
// Remove element at i.
copy(s.l[i:], s.l[i+1:])
s.l = s.l[:len(s.l)-1]
xi -= 1
} else {
x = c
xi = i
}
r -= c.Width
}
}
func (s *stream) samples() Samples {
samples := make(Samples, len(s.l))
copy(samples, s.l)
return samples
}

@ -1,7 +1,4 @@
Go support for Protocol Buffers - Google's data interchange format
Copyright 2010 The Go Authors. All rights reserved.
https://github.com/golang/protobuf
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are

@ -35,22 +35,39 @@
package proto
import (
"fmt"
"log"
"reflect"
"strings"
)
// Clone returns a deep copy of a protocol buffer.
func Clone(pb Message) Message {
in := reflect.ValueOf(pb)
func Clone(src Message) Message {
in := reflect.ValueOf(src)
if in.IsNil() {
return pb
return src
}
out := reflect.New(in.Type().Elem())
// out is empty so a merge is a deep copy.
mergeStruct(out.Elem(), in.Elem())
return out.Interface().(Message)
dst := out.Interface().(Message)
Merge(dst, src)
return dst
}
// Merger is the interface representing objects that can merge messages of the same type.
type Merger interface {
// Merge merges src into this message.
// Required and optional fields that are set in src will be set to that value in dst.
// Elements of repeated fields will be appended.
//
// Merge may panic if called with a different argument type than the receiver.
Merge(src Message)
}
// generatedMerger is the custom merge method that generated protos will have.
// We must add this method since a generate Merge method will conflict with
// many existing protos that have a Merge data field already defined.
type generatedMerger interface {
XXX_Merge(src Message)
}
// Merge merges src into dst.
@ -58,17 +75,24 @@ func Clone(pb Message) Message {
// Elements of repeated fields will be appended.
// Merge panics if src and dst are not the same type, or if dst is nil.
func Merge(dst, src Message) {
if m, ok := dst.(Merger); ok {
m.Merge(src)
return
}
in := reflect.ValueOf(src)
out := reflect.ValueOf(dst)
if out.IsNil() {
panic("proto: nil destination")
}
if in.Type() != out.Type() {
// Explicit test prior to mergeStruct so that mistyped nils will fail
panic("proto: type mismatch")
panic(fmt.Sprintf("proto.Merge(%T, %T) type mismatch", dst, src))
}
if in.IsNil() {
// Merging nil into non-nil is a quiet no-op
return // Merge from nil src is a noop
}
if m, ok := dst.(generatedMerger); ok {
m.XXX_Merge(src)
return
}
mergeStruct(out.Elem(), in.Elem())
@ -84,9 +108,15 @@ func mergeStruct(out, in reflect.Value) {
mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i])
}
if emIn, ok := in.Addr().Interface().(extendableProto); ok {
emOut := out.Addr().Interface().(extendableProto)
mergeExtension(emOut.ExtensionMap(), emIn.ExtensionMap())
if emIn, err := extendable(in.Addr().Interface()); err == nil {
emOut, _ := extendable(out.Addr().Interface())
mIn, muIn := emIn.extensionsRead()
if mIn != nil {
mOut := emOut.extensionsWrite()
muIn.Lock()
mergeExtension(mOut, mIn)
muIn.Unlock()
}
}
uf := in.FieldByName("XXX_unrecognized")

@ -39,8 +39,6 @@ import (
"errors"
"fmt"
"io"
"os"
"reflect"
)
// errOverflow is returned when an integer is too large to be represented.
@ -50,10 +48,6 @@ var errOverflow = errors.New("proto: integer overflow")
// wire type is encountered. It does not get returned to user code.
var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof")
// The fundamental decoders that interpret bytes on the wire.
// Those that take integer types all return uint64 and are
// therefore of type valueDecoder.
// DecodeVarint reads a varint-encoded integer from the slice.
// It returns the integer and the number of bytes consumed, or
// zero if there is not enough.
@ -61,7 +55,6 @@ var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for
// int32, int64, uint32, uint64, bool, and enum
// protocol buffer types.
func DecodeVarint(buf []byte) (x uint64, n int) {
// x, n already 0
for shift := uint(0); shift < 64; shift += 7 {
if n >= len(buf) {
return 0, 0
@ -78,13 +71,7 @@ func DecodeVarint(buf []byte) (x uint64, n int) {
return 0, 0
}
// DecodeVarint reads a varint-encoded integer from the Buffer.
// This is the format for the
// int32, int64, uint32, uint64, bool, and enum
// protocol buffer types.
func (p *Buffer) DecodeVarint() (x uint64, err error) {
// x, err already 0
func (p *Buffer) decodeVarintSlow() (x uint64, err error) {
i := p.index
l := len(p.buf)
@ -107,6 +94,107 @@ func (p *Buffer) DecodeVarint() (x uint64, err error) {
return
}
// DecodeVarint reads a varint-encoded integer from the Buffer.
// This is the format for the
// int32, int64, uint32, uint64, bool, and enum
// protocol buffer types.
func (p *Buffer) DecodeVarint() (x uint64, err error) {
i := p.index
buf := p.buf
if i >= len(buf) {
return 0, io.ErrUnexpectedEOF
} else if buf[i] < 0x80 {
p.index++
return uint64(buf[i]), nil
} else if len(buf)-i < 10 {
return p.decodeVarintSlow()
}
var b uint64
// we already checked the first byte
x = uint64(buf[i]) - 0x80
i++
b = uint64(buf[i])
i++
x += b << 7
if b&0x80 == 0 {
goto done
}
x -= 0x80 << 7
b = uint64(buf[i])
i++
x += b << 14
if b&0x80 == 0 {
goto done
}
x -= 0x80 << 14
b = uint64(buf[i])
i++
x += b << 21
if b&0x80 == 0 {
goto done
}
x -= 0x80 << 21
b = uint64(buf[i])
i++
x += b << 28
if b&0x80 == 0 {
goto done
}
x -= 0x80 << 28
b = uint64(buf[i])
i++
x += b << 35
if b&0x80 == 0 {
goto done
}
x -= 0x80 << 35
b = uint64(buf[i])
i++
x += b << 42
if b&0x80 == 0 {
goto done
}
x -= 0x80 << 42
b = uint64(buf[i])
i++
x += b << 49
if b&0x80 == 0 {
goto done
}
x -= 0x80 << 49
b = uint64(buf[i])
i++
x += b << 56
if b&0x80 == 0 {
goto done
}
x -= 0x80 << 56
b = uint64(buf[i])
i++
x += b << 63
if b&0x80 == 0 {
goto done
}
// x -= 0x80 << 63 // Always zero.
return 0, errOverflow
done:
p.index = i
return x, nil
}
// DecodeFixed64 reads a 64-bit integer from the Buffer.
// This is the format for the
// fixed64, sfixed64, and double protocol buffer types.
@ -173,9 +261,6 @@ func (p *Buffer) DecodeZigzag32() (x uint64, err error) {
return
}
// These are not ValueDecoders: they produce an array of bytes or a string.
// bytes, embedded messages
// DecodeRawBytes reads a count-delimited byte buffer from the Buffer.
// This is the format used for the bytes protocol buffer
// type and for embedded messages.
@ -217,81 +302,29 @@ func (p *Buffer) DecodeStringBytes() (s string, err error) {
return string(buf), nil
}
// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
// If the protocol buffer has extensions, and the field matches, add it as an extension.
// Otherwise, if the XXX_unrecognized field exists, append the skipped data there.
func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base structPointer, unrecField field) error {
oi := o.index
err := o.skip(t, tag, wire)
if err != nil {
return err
}
if !unrecField.IsValid() {
return nil
}
ptr := structPointer_Bytes(base, unrecField)
// Add the skipped field to struct field
obuf := o.buf
o.buf = *ptr
o.EncodeVarint(uint64(tag<<3 | wire))
*ptr = append(o.buf, obuf[oi:o.index]...)
o.buf = obuf
return nil
}
// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
func (o *Buffer) skip(t reflect.Type, tag, wire int) error {
var u uint64
var err error
switch wire {
case WireVarint:
_, err = o.DecodeVarint()
case WireFixed64:
_, err = o.DecodeFixed64()
case WireBytes:
_, err = o.DecodeRawBytes(false)
case WireFixed32:
_, err = o.DecodeFixed32()
case WireStartGroup:
for {
u, err = o.DecodeVarint()
if err != nil {
break
}
fwire := int(u & 0x7)
if fwire == WireEndGroup {
break
}
ftag := int(u >> 3)
err = o.skip(t, ftag, fwire)
if err != nil {
break
}
}
default:
err = fmt.Errorf("proto: can't skip unknown wire type %d for %s", wire, t)
}
return err
}
// Unmarshaler is the interface representing objects that can
// unmarshal themselves. The method should reset the receiver before
// decoding starts. The argument points to data that may be
// unmarshal themselves. The argument points to data that may be
// overwritten, so implementations should not keep references to the
// buffer.
// Unmarshal implementations should not clear the receiver.
// Any unmarshaled data should be merged into the receiver.
// Callers of Unmarshal that do not want to retain existing data
// should Reset the receiver before calling Unmarshal.
type Unmarshaler interface {
Unmarshal([]byte) error
}
// newUnmarshaler is the interface representing objects that can
// unmarshal themselves. The semantics are identical to Unmarshaler.
//
// This exists to support protoc-gen-go generated messages.
// The proto package will stop type-asserting to this interface in the future.
//
// DO NOT DEPEND ON THIS.
type newUnmarshaler interface {
XXX_Unmarshal([]byte) error
}
// Unmarshal parses the protocol buffer representation in buf and places the
// decoded result in pb. If the struct underlying pb does not match
// the data in buf, the results can be unpredictable.
@ -301,7 +334,13 @@ type Unmarshaler interface {
// to preserve and append to existing data.
func Unmarshal(buf []byte, pb Message) error {
pb.Reset()
return UnmarshalMerge(buf, pb)
if u, ok := pb.(newUnmarshaler); ok {
return u.XXX_Unmarshal(buf)
}
if u, ok := pb.(Unmarshaler); ok {
return u.Unmarshal(buf)
}
return NewBuffer(buf).Unmarshal(pb)
}
// UnmarshalMerge parses the protocol buffer representation in buf and
@ -311,8 +350,16 @@ func Unmarshal(buf []byte, pb Message) error {
// UnmarshalMerge merges into existing data in pb.
// Most code should use Unmarshal instead.
func UnmarshalMerge(buf []byte, pb Message) error {
// If the object can unmarshal itself, let it.
if u, ok := pb.(newUnmarshaler); ok {
return u.XXX_Unmarshal(buf)
}
if u, ok := pb.(Unmarshaler); ok {
// NOTE: The history of proto have unfortunately been inconsistent
// whether Unmarshaler should or should not implicitly clear itself.
// Some implementations do, most do not.
// Thus, calling this here may or may not do what people want.
//
// See https://github.com/golang/protobuf/issues/424
return u.Unmarshal(buf)
}
return NewBuffer(buf).Unmarshal(pb)
@ -328,541 +375,54 @@ func (p *Buffer) DecodeMessage(pb Message) error {
}
// DecodeGroup reads a tag-delimited group from the Buffer.
// StartGroup tag is already consumed. This function consumes
// EndGroup tag.
func (p *Buffer) DecodeGroup(pb Message) error {
typ, base, err := getbase(pb)
if err != nil {
return err
b := p.buf[p.index:]
x, y := findEndGroup(b)
if x < 0 {
return io.ErrUnexpectedEOF
}
return p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), true, base)
err := Unmarshal(b[:x], pb)
p.index += y
return err
}
// Unmarshal parses the protocol buffer representation in the
// Buffer and places the decoded result in pb. If the struct
// underlying pb does not match the data in the buffer, the results can be
// unpredictable.
//
// Unlike proto.Unmarshal, this does not reset pb before starting to unmarshal.
func (p *Buffer) Unmarshal(pb Message) error {
// If the object can unmarshal itself, let it.
if u, ok := pb.(Unmarshaler); ok {
err := u.Unmarshal(p.buf[p.index:])
if u, ok := pb.(newUnmarshaler); ok {
err := u.XXX_Unmarshal(p.buf[p.index:])
p.index = len(p.buf)
return err
}
typ, base, err := getbase(pb)
if err != nil {
return err
}
err = p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), false, base)
if collectStats {
stats.Decode++
}
return err
}
// unmarshalType does the work of unmarshaling a structure.
func (o *Buffer) unmarshalType(st reflect.Type, prop *StructProperties, is_group bool, base structPointer) error {
var state errorState
required, reqFields := prop.reqCount, uint64(0)
var err error
for err == nil && o.index < len(o.buf) {
oi := o.index
var u uint64
u, err = o.DecodeVarint()
if err != nil {
break
}
wire := int(u & 0x7)
if wire == WireEndGroup {
if is_group {
return nil // input is satisfied
}
return fmt.Errorf("proto: %s: wiretype end group for non-group", st)
}
tag := int(u >> 3)
if tag <= 0 {
return fmt.Errorf("proto: %s: illegal tag %d (wire type %d)", st, tag, wire)
}
fieldnum, ok := prop.decoderTags.get(tag)
if !ok {
// Maybe it's an extension?
if prop.extendable {
if e := structPointer_Interface(base, st).(extendableProto); isExtensionField(e, int32(tag)) {
if err = o.skip(st, tag, wire); err == nil {
ext := e.ExtensionMap()[int32(tag)] // may be missing
ext.enc = append(ext.enc, o.buf[oi:o.index]...)
e.ExtensionMap()[int32(tag)] = ext
}
continue
}
}
// Maybe it's a oneof?
if prop.oneofUnmarshaler != nil {
m := structPointer_Interface(base, st).(Message)
// First return value indicates whether tag is a oneof field.
ok, err = prop.oneofUnmarshaler(m, tag, wire, o)
if err == ErrInternalBadWireType {
// Map the error to something more descriptive.
// Do the formatting here to save generated code space.
err = fmt.Errorf("bad wiretype for oneof field in %T", m)
}
if ok {
continue
}
}
err = o.skipAndSave(st, tag, wire, base, prop.unrecField)
continue
}
p := prop.Prop[fieldnum]
if p.dec == nil {
fmt.Fprintf(os.Stderr, "proto: no protobuf decoder for %s.%s\n", st, st.Field(fieldnum).Name)
continue
}
dec := p.dec
if wire != WireStartGroup && wire != p.WireType {
if wire == WireBytes && p.packedDec != nil {
// a packable field
dec = p.packedDec
} else {
err = fmt.Errorf("proto: bad wiretype for field %s.%s: got wiretype %d, want %d", st, st.Field(fieldnum).Name, wire, p.WireType)
continue
}
}
decErr := dec(o, p, base)
if decErr != nil && !state.shouldContinue(decErr, p) {
err = decErr
}
if err == nil && p.Required {
// Successfully decoded a required field.
if tag <= 64 {
// use bitmap for fields 1-64 to catch field reuse.
var mask uint64 = 1 << uint64(tag-1)
if reqFields&mask == 0 {
// new required field
reqFields |= mask
required--
}
} else {
// This is imprecise. It can be fooled by a required field
// with a tag > 64 that is encoded twice; that's very rare.
// A fully correct implementation would require allocating
// a data structure, which we would like to avoid.
required--
}
}
}
if err == nil {
if is_group {
return io.ErrUnexpectedEOF
}
if state.err != nil {
return state.err
}
if required > 0 {
// Not enough information to determine the exact field. If we use extra
// CPU, we could determine the field only if the missing required field
// has a tag <= 64 and we check reqFields.
return &RequiredNotSetError{"{Unknown}"}
}
}
return err
}
// Individual type decoders
// For each,
// u is the decoded value,
// v is a pointer to the field (pointer) in the struct
// Sizes of the pools to allocate inside the Buffer.
// The goal is modest amortization and allocation
// on at least 16-byte boundaries.
const (
boolPoolSize = 16
uint32PoolSize = 8
uint64PoolSize = 4
)
// Decode a bool.
func (o *Buffer) dec_bool(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
if len(o.bools) == 0 {
o.bools = make([]bool, boolPoolSize)
}
o.bools[0] = u != 0
*structPointer_Bool(base, p.field) = &o.bools[0]
o.bools = o.bools[1:]
return nil
}