mirror of
https://github.com/prometheus-operator/prometheus-operator.git
synced 2025-04-21 19:49:46 +00:00
Update thanos to 0.4.0
Signed-off-by: Vasily <v.sliouniaev@gmail.com>
This commit is contained in:
parent
aa49a9946f
commit
6b4076cb18
93 changed files with 16551 additions and 149 deletions
go.modgo.summodules.txt
vendor
github.com
cespare/xxhash
LICENSE.txtREADME.mdgo.modgo.sumrotate.gorotate19.goxxhash.goxxhash_amd64.goxxhash_amd64.sxxhash_other.goxxhash_safe.goxxhash_unsafe.go
go-logfmt/logfmt
hashicorp
improbable-eng/thanos/pkg
oklog/ulid
.gitignore.travis.ymlAUTHORS.mdCHANGELOG.mdCONTRIBUTING.mdGopkg.lockGopkg.tomlLICENSEREADME.mdulid.go
prometheus
client_golang/prometheus
tsdb
.gitignore.travis.ymlCHANGELOG.mdLICENSEMAINTAINERS.mdMakefileMakefile.commonREADME.mdblock.gocheckpoint.go
chunkenc
chunks
compact.godb.goencoding
fileutil
dir_unix.godir_windows.gofileutil.goflock.goflock_plan9.goflock_solaris.goflock_unix.goflock_windows.gommap.gommap_386.gommap_amd64.gommap_unix.gommap_windows.gopreallocate.gopreallocate_darwin.gopreallocate_linux.gopreallocate_other.gosync.gosync_darwin.gosync_linux.go
go.modgo.sumhead.goindex
labels
querier.gorecord.gorepair.gotombstones.gowal.gowal
36
go.mod
36
go.mod
|
@ -4,53 +4,28 @@ go 1.12
|
|||
|
||||
require (
|
||||
github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
|
||||
github.com/blang/semver v3.5.1+incompatible
|
||||
github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d
|
||||
github.com/campoy/embedmd v1.0.0
|
||||
github.com/cespare/xxhash v1.1.0 // indirect
|
||||
github.com/emicklei/go-restful v2.6.0+incompatible // indirect
|
||||
github.com/evanphx/json-patch v4.1.0+incompatible // indirect
|
||||
github.com/fortytw2/leaktest v1.2.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.7 // indirect
|
||||
github.com/ghodss/yaml v1.0.0
|
||||
github.com/go-kit/kit v0.8.0
|
||||
github.com/go-logfmt/logfmt v0.3.0 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.0 // indirect
|
||||
github.com/go-openapi/jsonreference v0.19.0 // indirect
|
||||
github.com/go-openapi/spec v0.17.2
|
||||
github.com/go-openapi/swag v0.17.2 // indirect
|
||||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/gogo/protobuf v1.2.1 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4 // indirect
|
||||
github.com/golang/protobuf v1.2.0
|
||||
github.com/google/gofuzz v0.0.0-20150304233714-bbcb9da2d746 // indirect
|
||||
github.com/googleapis/gnostic v0.1.0 // indirect
|
||||
github.com/hashicorp/go-version v0.0.0-20171129150820-4fe82ae3040f
|
||||
github.com/hashicorp/golang-lru v0.5.0 // indirect
|
||||
github.com/hpcloud/tail v1.0.0 // indirect
|
||||
github.com/hashicorp/go-version v1.1.0
|
||||
github.com/imdario/mergo v0.3.5 // indirect
|
||||
github.com/improbable-eng/thanos v0.3.2
|
||||
github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3 // indirect
|
||||
github.com/improbable-eng/thanos v0.4.0
|
||||
github.com/jsonnet-bundler/jsonnet-bundler v0.1.0
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
|
||||
github.com/oklog/run v1.0.0
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/onsi/ginkgo v1.6.0 // indirect
|
||||
github.com/onsi/gomega v1.4.1 // indirect
|
||||
github.com/pkg/errors v0.8.1
|
||||
github.com/prometheus/client_golang v0.9.1
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect
|
||||
github.com/prometheus/common v0.0.0-20181218105931-67670fe90761 // indirect
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a // indirect
|
||||
github.com/prometheus/tsdb v0.2.0 // indirect
|
||||
github.com/spf13/pflag v1.0.3 // indirect
|
||||
github.com/prometheus/client_golang v0.9.2
|
||||
github.com/stretchr/testify v1.3.0
|
||||
golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d // indirect
|
||||
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect
|
||||
|
@ -61,10 +36,6 @@ require (
|
|||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262 // indirect
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/fsnotify.v1 v1.4.7 // indirect
|
||||
gopkg.in/inf.v0 v0.9.0 // indirect
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.2
|
||||
k8s.io/api v0.0.0-20190313235455-40a48860b5ab
|
||||
k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed
|
||||
|
@ -75,5 +46,4 @@ require (
|
|||
k8s.io/klog v0.3.0
|
||||
k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30
|
||||
k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 // indirect
|
||||
sigs.k8s.io/yaml v1.1.0 // indirect
|
||||
)
|
||||
|
|
204
go.sum
204
go.sum
|
@ -1,45 +1,81 @@
|
|||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
contrib.go.opencensus.io/exporter/ocagent v0.4.1/go.mod h1:b6YwD5Q3Yvj4yk0CDK5vGXexygNzI09aXUdDEakQBgA=
|
||||
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
|
||||
git.apache.org/thrift.git v0.0.0-20181218151757-9b75e4fe745a/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
|
||||
github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
|
||||
github.com/Azure/azure-sdk-for-go v23.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
|
||||
github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
|
||||
github.com/Azure/go-autorest v11.2.8+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
|
||||
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4=
|
||||
github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2 h1:CDDf61yprxfS7bmBPyhH8pxaobD2VbO3d7laAxJbZos=
|
||||
github.com/ant31/crd-validation v0.0.0-20180702145049-30f8a35d0ac2/go.mod h1:X0noFIik9YqfhGYBLEHg8LJKEwy7QIitLQuFMpKLcPk=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/aws/aws-sdk-go v0.0.0-20180507225419-00862f899353/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/biogo/store v0.0.0-20160505134755-913427a1d5e8/go.mod h1:Iev9Q3MErcn+w3UOJD/DkEzllvugfdx7bGcMOFhvr/4=
|
||||
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
|
||||
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d h1:DMb8SuAL9+demT8equqMMzD8C/uxqWmj4cgV7ufrpQo=
|
||||
github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0=
|
||||
github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
|
||||
github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
|
||||
github.com/cenk/backoff v2.0.0+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE=
|
||||
github.com/census-instrumentation/opencensus-proto v0.1.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
|
||||
github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4=
|
||||
github.com/cockroachdb/cockroach v0.0.0-20170608034007-84bc9597164f/go.mod h1:xeT/CQ0qZHangbYbWShlCGAx31aV4AjGswDUjhKS6HQ=
|
||||
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
|
||||
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
|
||||
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v0.0.0-20161101193935-9ed569b5d1ac/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/elastic/gosigar v0.9.0/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs=
|
||||
github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
|
||||
github.com/emicklei/go-restful v2.6.0+incompatible h1:luAX89wpjId5gV+GJV11MFD56GpAJTG2eUqCeDDgB98=
|
||||
github.com/emicklei/go-restful v2.6.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
|
||||
github.com/evanphx/json-patch v4.1.0+incompatible h1:K1MDoo4AZ4wU0GIU/fPmtZg7VpzLjCxu+UwBD1FvwOc=
|
||||
github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
|
||||
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA=
|
||||
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fatih/structtag v1.0.0/go.mod h1:IKitwq45uXL/yqi5mYghiD3w9H6eTOvI9vnk8tXMphA=
|
||||
github.com/fortytw2/leaktest v1.2.0 h1:cj6GCiwJDH7l3tMHLjZDo0QqPtrXJiWSI9JgpeQKw+Q=
|
||||
github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/getsentry/raven-go v0.1.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
|
||||
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-ini/ini v1.21.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
|
||||
github.com/go-openapi/jsonpointer v0.17.0 h1:nH6xp8XdXHx8dqveo0ZuJBluCO2qGrPbDNZ0dwoRHP0=
|
||||
github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
|
||||
github.com/go-openapi/jsonpointer v0.19.0 h1:FTUMcX77w5rQkClIzDtTxvn6Bsa894CcrzNj2MMfeg8=
|
||||
|
@ -54,34 +90,82 @@ github.com/go-openapi/swag v0.17.0 h1:iqrgMg7Q7SvtbWLlltPrkMs0UBJI6oTSs79JFRUi88
|
|||
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
|
||||
github.com/go-openapi/swag v0.17.2 h1:K/ycE/XTUDFltNHSO32cGRUhrVGJD64o8WgAIZNyc3k=
|
||||
github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
|
||||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4 h1:6UVLWz0fIIrv0UVj6t0A7cL48n8IyAdLVQqAYzEfsKI=
|
||||
github.com/golang/groupcache v0.0.0-20180924190550-6f2cf27854a4/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/snappy v0.0.0-20160529050041-d9eb7a3d35ec/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gofuzz v0.0.0-20150304233714-bbcb9da2d746 h1:M6d2zDTA4cKXT6OwFsJxlo5tWrAukj3KfvJ1zcBatnA=
|
||||
github.com/google/gofuzz v0.0.0-20150304233714-bbcb9da2d746/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20180605153948-8b03ce837f34/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gnostic v0.0.0-20180520015035-48a0ecefe2e4/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
|
||||
github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI=
|
||||
github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
|
||||
github.com/hashicorp/go-version v0.0.0-20171129150820-4fe82ae3040f h1:QnRipjW3Mm+sgD+vyO87cb+1RLzoM0mGVTYClil7mQg=
|
||||
github.com/hashicorp/go-version v0.0.0-20171129150820-4fe82ae3040f/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/gophercloud/gophercloud v0.0.0-20181206160319-9d88c34913a9/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20181025070259-68e3a13e4117/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.6.3/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
|
||||
github.com/hashicorp/consul v0.0.0-20180615161029-bed22a81e9fd/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.0.0-20160407174126-ad28ea4487f0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90/go.mod h1:o4zcYY1e0GEZI6eSEr+43QDYmuGglw1qSO6qdHUHCgg=
|
||||
github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
|
||||
github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE=
|
||||
github.com/hashicorp/serf v0.0.0-20161007004122-1d4fa605f6ff/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE=
|
||||
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
|
||||
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
|
||||
github.com/improbable-eng/thanos v0.3.2 h1:iZfU7exq+RD5Lnb8n3Eh9MNYoRLeyeGO/85AvEkLg+8=
|
||||
github.com/improbable-eng/thanos v0.3.2/go.mod h1:GZewVGILKuJVPNRn7L4Zw+7X96qzFOwj63b22xYGXBE=
|
||||
github.com/improbable-eng/thanos v0.4.0 h1:mIk6Pw0EPFLWW++zc9PFulvjdfZwQTqlTvbcO1JorCk=
|
||||
github.com/improbable-eng/thanos v0.4.0/go.mod h1:CYPZY0EwGQ4TTD4Y8Fh26yB923dGCoxslu70LAF2yWE=
|
||||
github.com/influxdata/influxdb v0.0.0-20170331210902-15e594fc09f1/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
|
||||
github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ=
|
||||
github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3 h1:/UewZcckqhvnnS0C6r3Sher2hSEbVmM6Ogpcjen08+Y=
|
||||
github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/jsonnet-bundler/jsonnet-bundler v0.1.0 h1:T/HtHFr+mYCRULrH1x/RnoB0prIs0rMkolJhFMXNC9A=
|
||||
github.com/jsonnet-bundler/jsonnet-bundler v0.1.0/go.mod h1:YKsSFc9VFhhLITkJS3X2PrRqWG9u2Jq99udTdDjQLfM=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v0.0.0-20150905172533-109e267447e9/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/julienschmidt/httprouter v1.1.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/knz/strtime v0.0.0-20181018220328-af2256ee352c/go.mod h1:4ZxfWkxwtc7dBeifERVVWRy9F9rTU9p0yCDgeCtlius=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
|
@ -91,43 +175,94 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
|||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
|
||||
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
|
||||
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lightstep/lightstep-tracer-go v0.15.6/go.mod h1:6AMpwZpsyCFwSovxzM78e+AsYxE8sGwiM6C3TytaWeI=
|
||||
github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt17nSPdS3K5e+ZTBY=
|
||||
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic=
|
||||
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.6 h1:SrwhHcpV4nWrMGdNcC2kXpMfcBVYGDuTArqyhocJgvA=
|
||||
github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1/go.mod h1:vuvdOZLJuf5HmJAJrKV64MmozrSsk+or0PB5dzdfspg=
|
||||
github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
|
||||
github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452 h1:hOY53G+kBFhbYFpRVxHl5eS7laP6B1+Cq+Z9Dry1iMU=
|
||||
github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
|
||||
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/mozillazg/go-cos v0.11.0/go.mod h1:Zp6DvvXn0RUOXGJ2chmWt2bLEqRAnJnS3DnAZsJsoaE=
|
||||
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/oklog/oklog v0.0.0-20170918173356-f857583a70c3/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
|
||||
github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
|
||||
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
|
||||
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.1 h1:PZSj/UFNaVp3KxrzHOcS7oyuWA7LoOY/77yCTEFu21U=
|
||||
github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
|
||||
github.com/opentracing-contrib/go-stdlib v0.0.0-20170113013457-1de4cc2120e7/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w=
|
||||
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
|
||||
github.com/opentracing/opentracing-go v1.0.1/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
|
||||
github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/peterbourgon/diskv v0.0.0-20180312054125-0646ccaebea1/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
|
||||
github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg=
|
||||
github.com/petermattis/goid v0.0.0-20170504144140-0ded85884ba5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.1 h1:K47Rk0v/fkEfwfQet2KWhscE0cJzjgCCDBG2KHZoVno=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181119215939-b36ad289a3ea/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181218105931-67670fe90761 h1:z6tvbDJ5OLJ48FFmnksv04a78maSTRBUIhkdHYV5Y98=
|
||||
github.com/prometheus/common v0.0.0-20181218105931-67670fe90761/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/tsdb v0.2.0 h1:27z98vFd/gPew17nmKEbLn37exGCwc2F5EyrgScg6bk=
|
||||
github.com/prometheus/tsdb v0.2.0/go.mod h1:lFf/o1J2a31WmWQbxYXfY1azJK5Xp5D8hwKMnVMBTGU=
|
||||
github.com/prometheus/prometheus v0.0.0-20190328180107-4d60eb36dcbe/go.mod h1:mVHnNN4pf1p9fM7dRvZQAKj0Ex93s+mIhtsml8SJy5E=
|
||||
github.com/prometheus/tsdb v0.6.1 h1:wUp5iY/rmwv6VNzZg4yaUVpkSs4nX5O5+aoviWTNeUo=
|
||||
github.com/prometheus/tsdb v0.6.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rlmcpherson/s3gof3r v0.5.0/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2r/Go7Bo71Rs=
|
||||
github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A=
|
||||
github.com/samuel/go-zookeeper v0.0.0-20161028232340-1d7be4effb13/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
|
||||
github.com/sasha-s/go-deadlock v0.0.0-20161201235124-341000892f3d/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
|
||||
github.com/shurcooL/vfsgen v0.0.0-20180711163814-62bca832be04/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
|
@ -137,71 +272,128 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
|
|||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.opencensus.io v0.19.0/go.mod h1:AYeH0+ZxYyghG8diqaaIq/9P3VgCCt5GF2ldCY4dkFg=
|
||||
golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190128193316-c7b33c32a30b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d h1:adrbvkTDn9rGnXg2IJDKozEpXXLZN89pdIA+Syt4/u0=
|
||||
golang.org/x/crypto v0.0.0-20190424203555-c05e17bb3b2d/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU=
|
||||
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
|
||||
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190425145619-16072639606e h1:4ktJgTV34+N3qOZUc5fAaG3Pb11qzMm3PkAoTAgUZ2I=
|
||||
golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1 h1:nsUiJHvm6yOoRozW9Tz0siNk9sHieLzR+w814Ihse3A=
|
||||
golang.org/x/text v0.3.1/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181023010539-40a48ad93fbe/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262 h1:qsl9y/CJx34tuA7QCPNp86JNJe4spst6Ff8MjvPUdPg=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/fsnotify/fsnotify.v1 v1.3.0/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE=
|
||||
gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE=
|
||||
gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o=
|
||||
gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/vmihailenco/msgpack.v2 v2.9.1/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8=
|
||||
gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
k8s.io/api v0.0.0-20181213150558-05914d821849/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
|
||||
k8s.io/api v0.0.0-20190313235455-40a48860b5ab h1:DG9A67baNpoeweOy2spF1OWHhnVY5KR7/Ek/+U1lVZc=
|
||||
k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
|
||||
k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed h1:rCteec//ELIjZMfjIGQbVtZooyaofqDJwsmWwWKItNs=
|
||||
k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE=
|
||||
k8s.io/apimachinery v0.0.0-20181127025237-2b1284ed4c93/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
|
||||
k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1 h1:IS7K02iBkQXpCeieSiyJjGoLSdVOv2DbPaWHJ+ZtgKg=
|
||||
k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
|
||||
k8s.io/client-go v2.0.0-alpha.0.0.20181121191925-a47917edff34+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
|
||||
k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o=
|
||||
k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
|
||||
k8s.io/code-generator v0.0.0-20190311093542-50b561225d70 h1:lgPp615xLHxN84RBd+viA/oHzJfI0miFYFH4T9wpPQ4=
|
||||
k8s.io/code-generator v0.0.0-20190311093542-50b561225d70/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8=
|
||||
k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a h1:QoHVuRquf80YZ+/bovwxoMO3Q/A3nt3yTgS0/0nejuk=
|
||||
k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
|
||||
k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
|
||||
k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE=
|
||||
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
|
||||
k8s.io/kube-openapi v0.0.0-20180629012420-d83b052f768a/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
|
||||
k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI=
|
||||
k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
|
||||
k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 h1:8r+l4bNWjRlsFYlQJnKJ2p7s1YQPj4XyXiJVqDHRx7c=
|
||||
k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
|
||||
labix.org/v2/mgo v0.0.0-20140701140051-000000000287/go.mod h1:Lg7AYkt1uXJoR9oeSZ3W/8IXLdvOfIITgZnommstyz4=
|
||||
launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM=
|
||||
sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
|
||||
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
|
||||
|
|
22
vendor/github.com/cespare/xxhash/LICENSE.txt
generated
vendored
Normal file
22
vendor/github.com/cespare/xxhash/LICENSE.txt
generated
vendored
Normal file
|
@ -0,0 +1,22 @@
|
|||
Copyright (c) 2016 Caleb Spare
|
||||
|
||||
MIT License
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
50
vendor/github.com/cespare/xxhash/README.md
generated
vendored
Normal file
50
vendor/github.com/cespare/xxhash/README.md
generated
vendored
Normal file
|
@ -0,0 +1,50 @@
|
|||
# xxhash
|
||||
|
||||
[](https://godoc.org/github.com/cespare/xxhash)
|
||||
|
||||
xxhash is a Go implementation of the 64-bit
|
||||
[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a
|
||||
high-quality hashing algorithm that is much faster than anything in the Go
|
||||
standard library.
|
||||
|
||||
The API is very small, taking its cue from the other hashing packages in the
|
||||
standard library:
|
||||
|
||||
$ go doc github.com/cespare/xxhash !
|
||||
package xxhash // import "github.com/cespare/xxhash"
|
||||
|
||||
Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
|
||||
at http://cyan4973.github.io/xxHash/.
|
||||
|
||||
func New() hash.Hash64
|
||||
func Sum64(b []byte) uint64
|
||||
func Sum64String(s string) uint64
|
||||
|
||||
This implementation provides a fast pure-Go implementation and an even faster
|
||||
assembly implementation for amd64.
|
||||
|
||||
## Benchmarks
|
||||
|
||||
Here are some quick benchmarks comparing the pure-Go and assembly
|
||||
implementations of Sum64 against another popular Go XXH64 implementation,
|
||||
[github.com/OneOfOne/xxhash](https://github.com/OneOfOne/xxhash):
|
||||
|
||||
| input size | OneOfOne | cespare (purego) | cespare |
|
||||
| --- | --- | --- | --- |
|
||||
| 5 B | 416 MB/s | 720 MB/s | 872 MB/s |
|
||||
| 100 B | 3980 MB/s | 5013 MB/s | 5252 MB/s |
|
||||
| 4 KB | 12727 MB/s | 12999 MB/s | 13026 MB/s |
|
||||
| 10 MB | 9879 MB/s | 10775 MB/s | 10913 MB/s |
|
||||
|
||||
These numbers were generated with:
|
||||
|
||||
```
|
||||
$ go test -benchtime 10s -bench '/OneOfOne,'
|
||||
$ go test -tags purego -benchtime 10s -bench '/xxhash,'
|
||||
$ go test -benchtime 10s -bench '/xxhash,'
|
||||
```
|
||||
|
||||
## Projects using this package
|
||||
|
||||
- [InfluxDB](https://github.com/influxdata/influxdb)
|
||||
- [Prometheus](https://github.com/prometheus/prometheus)
|
6
vendor/github.com/cespare/xxhash/go.mod
generated
vendored
Normal file
6
vendor/github.com/cespare/xxhash/go.mod
generated
vendored
Normal file
|
@ -0,0 +1,6 @@
|
|||
module github.com/cespare/xxhash
|
||||
|
||||
require (
|
||||
github.com/OneOfOne/xxhash v1.2.2
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72
|
||||
)
|
4
vendor/github.com/cespare/xxhash/go.sum
generated
vendored
Normal file
4
vendor/github.com/cespare/xxhash/go.sum
generated
vendored
Normal file
|
@ -0,0 +1,4 @@
|
|||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
14
vendor/github.com/cespare/xxhash/rotate.go
generated
vendored
Normal file
14
vendor/github.com/cespare/xxhash/rotate.go
generated
vendored
Normal file
|
@ -0,0 +1,14 @@
|
|||
// +build !go1.9
|
||||
|
||||
package xxhash
|
||||
|
||||
// TODO(caleb): After Go 1.10 comes out, remove this fallback code.
|
||||
|
||||
func rol1(x uint64) uint64 { return (x << 1) | (x >> (64 - 1)) }
|
||||
func rol7(x uint64) uint64 { return (x << 7) | (x >> (64 - 7)) }
|
||||
func rol11(x uint64) uint64 { return (x << 11) | (x >> (64 - 11)) }
|
||||
func rol12(x uint64) uint64 { return (x << 12) | (x >> (64 - 12)) }
|
||||
func rol18(x uint64) uint64 { return (x << 18) | (x >> (64 - 18)) }
|
||||
func rol23(x uint64) uint64 { return (x << 23) | (x >> (64 - 23)) }
|
||||
func rol27(x uint64) uint64 { return (x << 27) | (x >> (64 - 27)) }
|
||||
func rol31(x uint64) uint64 { return (x << 31) | (x >> (64 - 31)) }
|
14
vendor/github.com/cespare/xxhash/rotate19.go
generated
vendored
Normal file
14
vendor/github.com/cespare/xxhash/rotate19.go
generated
vendored
Normal file
|
@ -0,0 +1,14 @@
|
|||
// +build go1.9
|
||||
|
||||
package xxhash
|
||||
|
||||
import "math/bits"
|
||||
|
||||
func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) }
|
||||
func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) }
|
||||
func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) }
|
||||
func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) }
|
||||
func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) }
|
||||
func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) }
|
||||
func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) }
|
||||
func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) }
|
168
vendor/github.com/cespare/xxhash/xxhash.go
generated
vendored
Normal file
168
vendor/github.com/cespare/xxhash/xxhash.go
generated
vendored
Normal file
|
@ -0,0 +1,168 @@
|
|||
// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described
|
||||
// at http://cyan4973.github.io/xxHash/.
|
||||
package xxhash
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash"
|
||||
)
|
||||
|
||||
const (
|
||||
prime1 uint64 = 11400714785074694791
|
||||
prime2 uint64 = 14029467366897019727
|
||||
prime3 uint64 = 1609587929392839161
|
||||
prime4 uint64 = 9650029242287828579
|
||||
prime5 uint64 = 2870177450012600261
|
||||
)
|
||||
|
||||
// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where
|
||||
// possible in the Go code is worth a small (but measurable) performance boost
|
||||
// by avoiding some MOVQs. Vars are needed for the asm and also are useful for
|
||||
// convenience in the Go code in a few places where we need to intentionally
|
||||
// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the
|
||||
// result overflows a uint64).
|
||||
var (
|
||||
prime1v = prime1
|
||||
prime2v = prime2
|
||||
prime3v = prime3
|
||||
prime4v = prime4
|
||||
prime5v = prime5
|
||||
)
|
||||
|
||||
type xxh struct {
|
||||
v1 uint64
|
||||
v2 uint64
|
||||
v3 uint64
|
||||
v4 uint64
|
||||
total int
|
||||
mem [32]byte
|
||||
n int // how much of mem is used
|
||||
}
|
||||
|
||||
// New creates a new hash.Hash64 that implements the 64-bit xxHash algorithm.
|
||||
func New() hash.Hash64 {
|
||||
var x xxh
|
||||
x.Reset()
|
||||
return &x
|
||||
}
|
||||
|
||||
func (x *xxh) Reset() {
|
||||
x.n = 0
|
||||
x.total = 0
|
||||
x.v1 = prime1v + prime2
|
||||
x.v2 = prime2
|
||||
x.v3 = 0
|
||||
x.v4 = -prime1v
|
||||
}
|
||||
|
||||
func (x *xxh) Size() int { return 8 }
|
||||
func (x *xxh) BlockSize() int { return 32 }
|
||||
|
||||
// Write adds more data to x. It always returns len(b), nil.
|
||||
func (x *xxh) Write(b []byte) (n int, err error) {
|
||||
n = len(b)
|
||||
x.total += len(b)
|
||||
|
||||
if x.n+len(b) < 32 {
|
||||
// This new data doesn't even fill the current block.
|
||||
copy(x.mem[x.n:], b)
|
||||
x.n += len(b)
|
||||
return
|
||||
}
|
||||
|
||||
if x.n > 0 {
|
||||
// Finish off the partial block.
|
||||
copy(x.mem[x.n:], b)
|
||||
x.v1 = round(x.v1, u64(x.mem[0:8]))
|
||||
x.v2 = round(x.v2, u64(x.mem[8:16]))
|
||||
x.v3 = round(x.v3, u64(x.mem[16:24]))
|
||||
x.v4 = round(x.v4, u64(x.mem[24:32]))
|
||||
b = b[32-x.n:]
|
||||
x.n = 0
|
||||
}
|
||||
|
||||
if len(b) >= 32 {
|
||||
// One or more full blocks left.
|
||||
b = writeBlocks(x, b)
|
||||
}
|
||||
|
||||
// Store any remaining partial block.
|
||||
copy(x.mem[:], b)
|
||||
x.n = len(b)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (x *xxh) Sum(b []byte) []byte {
|
||||
s := x.Sum64()
|
||||
return append(
|
||||
b,
|
||||
byte(s>>56),
|
||||
byte(s>>48),
|
||||
byte(s>>40),
|
||||
byte(s>>32),
|
||||
byte(s>>24),
|
||||
byte(s>>16),
|
||||
byte(s>>8),
|
||||
byte(s),
|
||||
)
|
||||
}
|
||||
|
||||
func (x *xxh) Sum64() uint64 {
|
||||
var h uint64
|
||||
|
||||
if x.total >= 32 {
|
||||
v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4
|
||||
h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
|
||||
h = mergeRound(h, v1)
|
||||
h = mergeRound(h, v2)
|
||||
h = mergeRound(h, v3)
|
||||
h = mergeRound(h, v4)
|
||||
} else {
|
||||
h = x.v3 + prime5
|
||||
}
|
||||
|
||||
h += uint64(x.total)
|
||||
|
||||
i, end := 0, x.n
|
||||
for ; i+8 <= end; i += 8 {
|
||||
k1 := round(0, u64(x.mem[i:i+8]))
|
||||
h ^= k1
|
||||
h = rol27(h)*prime1 + prime4
|
||||
}
|
||||
if i+4 <= end {
|
||||
h ^= uint64(u32(x.mem[i:i+4])) * prime1
|
||||
h = rol23(h)*prime2 + prime3
|
||||
i += 4
|
||||
}
|
||||
for i < end {
|
||||
h ^= uint64(x.mem[i]) * prime5
|
||||
h = rol11(h) * prime1
|
||||
i++
|
||||
}
|
||||
|
||||
h ^= h >> 33
|
||||
h *= prime2
|
||||
h ^= h >> 29
|
||||
h *= prime3
|
||||
h ^= h >> 32
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) }
|
||||
func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) }
|
||||
|
||||
func round(acc, input uint64) uint64 {
|
||||
acc += input * prime2
|
||||
acc = rol31(acc)
|
||||
acc *= prime1
|
||||
return acc
|
||||
}
|
||||
|
||||
func mergeRound(acc, val uint64) uint64 {
|
||||
val = round(0, val)
|
||||
acc ^= val
|
||||
acc = acc*prime1 + prime4
|
||||
return acc
|
||||
}
|
12
vendor/github.com/cespare/xxhash/xxhash_amd64.go
generated
vendored
Normal file
12
vendor/github.com/cespare/xxhash/xxhash_amd64.go
generated
vendored
Normal file
|
@ -0,0 +1,12 @@
|
|||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !purego
|
||||
|
||||
package xxhash
|
||||
|
||||
// Sum64 computes the 64-bit xxHash digest of b.
|
||||
//
|
||||
//go:noescape
|
||||
func Sum64(b []byte) uint64
|
||||
|
||||
func writeBlocks(x *xxh, b []byte) []byte
|
233
vendor/github.com/cespare/xxhash/xxhash_amd64.s
generated
vendored
Normal file
233
vendor/github.com/cespare/xxhash/xxhash_amd64.s
generated
vendored
Normal file
|
@ -0,0 +1,233 @@
|
|||
// +build !appengine
|
||||
// +build gc
|
||||
// +build !purego
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// Register allocation:
|
||||
// AX h
|
||||
// CX pointer to advance through b
|
||||
// DX n
|
||||
// BX loop end
|
||||
// R8 v1, k1
|
||||
// R9 v2
|
||||
// R10 v3
|
||||
// R11 v4
|
||||
// R12 tmp
|
||||
// R13 prime1v
|
||||
// R14 prime2v
|
||||
// R15 prime4v
|
||||
|
||||
// round reads from and advances the buffer pointer in CX.
|
||||
// It assumes that R13 has prime1v and R14 has prime2v.
|
||||
#define round(r) \
|
||||
MOVQ (CX), R12 \
|
||||
ADDQ $8, CX \
|
||||
IMULQ R14, R12 \
|
||||
ADDQ R12, r \
|
||||
ROLQ $31, r \
|
||||
IMULQ R13, r
|
||||
|
||||
// mergeRound applies a merge round on the two registers acc and val.
|
||||
// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v.
|
||||
#define mergeRound(acc, val) \
|
||||
IMULQ R14, val \
|
||||
ROLQ $31, val \
|
||||
IMULQ R13, val \
|
||||
XORQ val, acc \
|
||||
IMULQ R13, acc \
|
||||
ADDQ R15, acc
|
||||
|
||||
// func Sum64(b []byte) uint64
|
||||
TEXT ·Sum64(SB), NOSPLIT, $0-32
|
||||
// Load fixed primes.
|
||||
MOVQ ·prime1v(SB), R13
|
||||
MOVQ ·prime2v(SB), R14
|
||||
MOVQ ·prime4v(SB), R15
|
||||
|
||||
// Load slice.
|
||||
MOVQ b_base+0(FP), CX
|
||||
MOVQ b_len+8(FP), DX
|
||||
LEAQ (CX)(DX*1), BX
|
||||
|
||||
// The first loop limit will be len(b)-32.
|
||||
SUBQ $32, BX
|
||||
|
||||
// Check whether we have at least one block.
|
||||
CMPQ DX, $32
|
||||
JLT noBlocks
|
||||
|
||||
// Set up initial state (v1, v2, v3, v4).
|
||||
MOVQ R13, R8
|
||||
ADDQ R14, R8
|
||||
MOVQ R14, R9
|
||||
XORQ R10, R10
|
||||
XORQ R11, R11
|
||||
SUBQ R13, R11
|
||||
|
||||
// Loop until CX > BX.
|
||||
blockLoop:
|
||||
round(R8)
|
||||
round(R9)
|
||||
round(R10)
|
||||
round(R11)
|
||||
|
||||
CMPQ CX, BX
|
||||
JLE blockLoop
|
||||
|
||||
MOVQ R8, AX
|
||||
ROLQ $1, AX
|
||||
MOVQ R9, R12
|
||||
ROLQ $7, R12
|
||||
ADDQ R12, AX
|
||||
MOVQ R10, R12
|
||||
ROLQ $12, R12
|
||||
ADDQ R12, AX
|
||||
MOVQ R11, R12
|
||||
ROLQ $18, R12
|
||||
ADDQ R12, AX
|
||||
|
||||
mergeRound(AX, R8)
|
||||
mergeRound(AX, R9)
|
||||
mergeRound(AX, R10)
|
||||
mergeRound(AX, R11)
|
||||
|
||||
JMP afterBlocks
|
||||
|
||||
noBlocks:
|
||||
MOVQ ·prime5v(SB), AX
|
||||
|
||||
afterBlocks:
|
||||
ADDQ DX, AX
|
||||
|
||||
// Right now BX has len(b)-32, and we want to loop until CX > len(b)-8.
|
||||
ADDQ $24, BX
|
||||
|
||||
CMPQ CX, BX
|
||||
JG fourByte
|
||||
|
||||
wordLoop:
|
||||
// Calculate k1.
|
||||
MOVQ (CX), R8
|
||||
ADDQ $8, CX
|
||||
IMULQ R14, R8
|
||||
ROLQ $31, R8
|
||||
IMULQ R13, R8
|
||||
|
||||
XORQ R8, AX
|
||||
ROLQ $27, AX
|
||||
IMULQ R13, AX
|
||||
ADDQ R15, AX
|
||||
|
||||
CMPQ CX, BX
|
||||
JLE wordLoop
|
||||
|
||||
fourByte:
|
||||
ADDQ $4, BX
|
||||
CMPQ CX, BX
|
||||
JG singles
|
||||
|
||||
MOVL (CX), R8
|
||||
ADDQ $4, CX
|
||||
IMULQ R13, R8
|
||||
XORQ R8, AX
|
||||
|
||||
ROLQ $23, AX
|
||||
IMULQ R14, AX
|
||||
ADDQ ·prime3v(SB), AX
|
||||
|
||||
singles:
|
||||
ADDQ $4, BX
|
||||
CMPQ CX, BX
|
||||
JGE finalize
|
||||
|
||||
singlesLoop:
|
||||
MOVBQZX (CX), R12
|
||||
ADDQ $1, CX
|
||||
IMULQ ·prime5v(SB), R12
|
||||
XORQ R12, AX
|
||||
|
||||
ROLQ $11, AX
|
||||
IMULQ R13, AX
|
||||
|
||||
CMPQ CX, BX
|
||||
JL singlesLoop
|
||||
|
||||
finalize:
|
||||
MOVQ AX, R12
|
||||
SHRQ $33, R12
|
||||
XORQ R12, AX
|
||||
IMULQ R14, AX
|
||||
MOVQ AX, R12
|
||||
SHRQ $29, R12
|
||||
XORQ R12, AX
|
||||
IMULQ ·prime3v(SB), AX
|
||||
MOVQ AX, R12
|
||||
SHRQ $32, R12
|
||||
XORQ R12, AX
|
||||
|
||||
MOVQ AX, ret+24(FP)
|
||||
RET
|
||||
|
||||
// writeBlocks uses the same registers as above except that it uses AX to store
|
||||
// the x pointer.
|
||||
|
||||
// func writeBlocks(x *xxh, b []byte) []byte
|
||||
TEXT ·writeBlocks(SB), NOSPLIT, $0-56
|
||||
// Load fixed primes needed for round.
|
||||
MOVQ ·prime1v(SB), R13
|
||||
MOVQ ·prime2v(SB), R14
|
||||
|
||||
// Load slice.
|
||||
MOVQ b_base+8(FP), CX
|
||||
MOVQ CX, ret_base+32(FP) // initialize return base pointer; see NOTE below
|
||||
MOVQ b_len+16(FP), DX
|
||||
LEAQ (CX)(DX*1), BX
|
||||
SUBQ $32, BX
|
||||
|
||||
// Load vN from x.
|
||||
MOVQ x+0(FP), AX
|
||||
MOVQ 0(AX), R8 // v1
|
||||
MOVQ 8(AX), R9 // v2
|
||||
MOVQ 16(AX), R10 // v3
|
||||
MOVQ 24(AX), R11 // v4
|
||||
|
||||
// We don't need to check the loop condition here; this function is
|
||||
// always called with at least one block of data to process.
|
||||
blockLoop:
|
||||
round(R8)
|
||||
round(R9)
|
||||
round(R10)
|
||||
round(R11)
|
||||
|
||||
CMPQ CX, BX
|
||||
JLE blockLoop
|
||||
|
||||
// Copy vN back to x.
|
||||
MOVQ R8, 0(AX)
|
||||
MOVQ R9, 8(AX)
|
||||
MOVQ R10, 16(AX)
|
||||
MOVQ R11, 24(AX)
|
||||
|
||||
// Construct return slice.
|
||||
// NOTE: It's important that we don't construct a slice that has a base
|
||||
// pointer off the end of the original slice, as in Go 1.7+ this will
|
||||
// cause runtime crashes. (See discussion in, for example,
|
||||
// https://github.com/golang/go/issues/16772.)
|
||||
// Therefore, we calculate the length/cap first, and if they're zero, we
|
||||
// keep the old base. This is what the compiler does as well if you
|
||||
// write code like
|
||||
// b = b[len(b):]
|
||||
|
||||
// New length is 32 - (CX - BX) -> BX+32 - CX.
|
||||
ADDQ $32, BX
|
||||
SUBQ CX, BX
|
||||
JZ afterSetBase
|
||||
|
||||
MOVQ CX, ret_base+32(FP)
|
||||
|
||||
afterSetBase:
|
||||
MOVQ BX, ret_len+40(FP)
|
||||
MOVQ BX, ret_cap+48(FP) // set cap == len
|
||||
|
||||
RET
|
75
vendor/github.com/cespare/xxhash/xxhash_other.go
generated
vendored
Normal file
75
vendor/github.com/cespare/xxhash/xxhash_other.go
generated
vendored
Normal file
|
@ -0,0 +1,75 @@
|
|||
// +build !amd64 appengine !gc purego
|
||||
|
||||
package xxhash
|
||||
|
||||
// Sum64 computes the 64-bit xxHash digest of b.
|
||||
func Sum64(b []byte) uint64 {
|
||||
// A simpler version would be
|
||||
// x := New()
|
||||
// x.Write(b)
|
||||
// return x.Sum64()
|
||||
// but this is faster, particularly for small inputs.
|
||||
|
||||
n := len(b)
|
||||
var h uint64
|
||||
|
||||
if n >= 32 {
|
||||
v1 := prime1v + prime2
|
||||
v2 := prime2
|
||||
v3 := uint64(0)
|
||||
v4 := -prime1v
|
||||
for len(b) >= 32 {
|
||||
v1 = round(v1, u64(b[0:8:len(b)]))
|
||||
v2 = round(v2, u64(b[8:16:len(b)]))
|
||||
v3 = round(v3, u64(b[16:24:len(b)]))
|
||||
v4 = round(v4, u64(b[24:32:len(b)]))
|
||||
b = b[32:len(b):len(b)]
|
||||
}
|
||||
h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
|
||||
h = mergeRound(h, v1)
|
||||
h = mergeRound(h, v2)
|
||||
h = mergeRound(h, v3)
|
||||
h = mergeRound(h, v4)
|
||||
} else {
|
||||
h = prime5
|
||||
}
|
||||
|
||||
h += uint64(n)
|
||||
|
||||
i, end := 0, len(b)
|
||||
for ; i+8 <= end; i += 8 {
|
||||
k1 := round(0, u64(b[i:i+8:len(b)]))
|
||||
h ^= k1
|
||||
h = rol27(h)*prime1 + prime4
|
||||
}
|
||||
if i+4 <= end {
|
||||
h ^= uint64(u32(b[i:i+4:len(b)])) * prime1
|
||||
h = rol23(h)*prime2 + prime3
|
||||
i += 4
|
||||
}
|
||||
for ; i < end; i++ {
|
||||
h ^= uint64(b[i]) * prime5
|
||||
h = rol11(h) * prime1
|
||||
}
|
||||
|
||||
h ^= h >> 33
|
||||
h *= prime2
|
||||
h ^= h >> 29
|
||||
h *= prime3
|
||||
h ^= h >> 32
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
func writeBlocks(x *xxh, b []byte) []byte {
|
||||
v1, v2, v3, v4 := x.v1, x.v2, x.v3, x.v4
|
||||
for len(b) >= 32 {
|
||||
v1 = round(v1, u64(b[0:8:len(b)]))
|
||||
v2 = round(v2, u64(b[8:16:len(b)]))
|
||||
v3 = round(v3, u64(b[16:24:len(b)]))
|
||||
v4 = round(v4, u64(b[24:32:len(b)]))
|
||||
b = b[32:len(b):len(b)]
|
||||
}
|
||||
x.v1, x.v2, x.v3, x.v4 = v1, v2, v3, v4
|
||||
return b
|
||||
}
|
10
vendor/github.com/cespare/xxhash/xxhash_safe.go
generated
vendored
Normal file
10
vendor/github.com/cespare/xxhash/xxhash_safe.go
generated
vendored
Normal file
|
@ -0,0 +1,10 @@
|
|||
// +build appengine
|
||||
|
||||
// This file contains the safe implementations of otherwise unsafe-using code.
|
||||
|
||||
package xxhash
|
||||
|
||||
// Sum64String computes the 64-bit xxHash digest of s.
|
||||
func Sum64String(s string) uint64 {
|
||||
return Sum64([]byte(s))
|
||||
}
|
30
vendor/github.com/cespare/xxhash/xxhash_unsafe.go
generated
vendored
Normal file
30
vendor/github.com/cespare/xxhash/xxhash_unsafe.go
generated
vendored
Normal file
|
@ -0,0 +1,30 @@
|
|||
// +build !appengine
|
||||
|
||||
// This file encapsulates usage of unsafe.
|
||||
// xxhash_safe.go contains the safe implementations.
|
||||
|
||||
package xxhash
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Sum64String computes the 64-bit xxHash digest of s.
|
||||
// It may be faster than Sum64([]byte(s)) by avoiding a copy.
|
||||
//
|
||||
// TODO(caleb): Consider removing this if an optimization is ever added to make
|
||||
// it unnecessary: https://golang.org/issue/2205.
|
||||
//
|
||||
// TODO(caleb): We still have a function call; we could instead write Go/asm
|
||||
// copies of Sum64 for strings to squeeze out a bit more speed.
|
||||
func Sum64String(s string) uint64 {
|
||||
// See https://groups.google.com/d/msg/golang-nuts/dcjzJy-bSpw/tcZYBzQqAQAJ
|
||||
// for some discussion about this unsafe conversion.
|
||||
var b []byte
|
||||
bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
||||
bh.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
|
||||
bh.Len = len(s)
|
||||
bh.Cap = len(s)
|
||||
return Sum64(b)
|
||||
}
|
11
vendor/github.com/go-logfmt/logfmt/.travis.yml
generated
vendored
11
vendor/github.com/go-logfmt/logfmt/.travis.yml
generated
vendored
|
@ -1,11 +1,12 @@
|
|||
language: go
|
||||
sudo: false
|
||||
go:
|
||||
- 1.3
|
||||
- 1.4
|
||||
- 1.5
|
||||
- 1.6
|
||||
- tip
|
||||
- "1.7.x"
|
||||
- "1.8.x"
|
||||
- "1.9.x"
|
||||
- "1.10.x"
|
||||
- "1.11.x"
|
||||
- "tip"
|
||||
|
||||
before_install:
|
||||
- go get github.com/mattn/goveralls
|
||||
|
|
41
vendor/github.com/go-logfmt/logfmt/CHANGELOG.md
generated
vendored
Normal file
41
vendor/github.com/go-logfmt/logfmt/CHANGELOG.md
generated
vendored
Normal file
|
@ -0,0 +1,41 @@
|
|||
# Changelog
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [0.4.0] - 2018-11-21
|
||||
|
||||
### Added
|
||||
- Go module support by [@ChrisHines]
|
||||
- CHANGELOG by [@ChrisHines]
|
||||
|
||||
### Changed
|
||||
- Drop invalid runes from keys instead of returning ErrInvalidKey by [@ChrisHines]
|
||||
- On panic while printing, attempt to print panic value by [@bboreham]
|
||||
|
||||
## [0.3.0] - 2016-11-15
|
||||
### Added
|
||||
- Pool buffers for quoted strings and byte slices by [@nussjustin]
|
||||
### Fixed
|
||||
- Fuzz fix, quote invalid UTF-8 values by [@judwhite]
|
||||
|
||||
## [0.2.0] - 2016-05-08
|
||||
### Added
|
||||
- Encoder.EncodeKeyvals by [@ChrisHines]
|
||||
|
||||
## [0.1.0] - 2016-03-28
|
||||
### Added
|
||||
- Encoder by [@ChrisHines]
|
||||
- Decoder by [@ChrisHines]
|
||||
- MarshalKeyvals by [@ChrisHines]
|
||||
|
||||
[0.4.0]: https://github.com/go-logfmt/logfmt/compare/v0.3.0...v0.4.0
|
||||
[0.3.0]: https://github.com/go-logfmt/logfmt/compare/v0.2.0...v0.3.0
|
||||
[0.2.0]: https://github.com/go-logfmt/logfmt/compare/v0.1.0...v0.2.0
|
||||
[0.1.0]: https://github.com/go-logfmt/logfmt/commits/v0.1.0
|
||||
|
||||
[@ChrisHines]: https://github.com/ChrisHines
|
||||
[@bboreham]: https://github.com/bboreham
|
||||
[@judwhite]: https://github.com/judwhite
|
||||
[@nussjustin]: https://github.com/nussjustin
|
39
vendor/github.com/go-logfmt/logfmt/encode.go
generated
vendored
39
vendor/github.com/go-logfmt/logfmt/encode.go
generated
vendored
|
@ -110,8 +110,8 @@ func (e *MarshalerError) Error() string {
|
|||
// a nil interface or pointer value.
|
||||
var ErrNilKey = errors.New("nil key")
|
||||
|
||||
// ErrInvalidKey is returned by Marshal functions and Encoder methods if a key
|
||||
// contains an invalid character.
|
||||
// ErrInvalidKey is returned by Marshal functions and Encoder methods if, after
|
||||
// dropping invalid runes, a key is empty.
|
||||
var ErrInvalidKey = errors.New("invalid key")
|
||||
|
||||
// ErrUnsupportedKeyType is returned by Encoder methods if a key has an
|
||||
|
@ -165,31 +165,32 @@ func writeKey(w io.Writer, key interface{}) error {
|
|||
}
|
||||
}
|
||||
|
||||
func invalidKeyRune(r rune) bool {
|
||||
return r <= ' ' || r == '=' || r == '"' || r == utf8.RuneError
|
||||
}
|
||||
|
||||
func invalidKeyString(key string) bool {
|
||||
return len(key) == 0 || strings.IndexFunc(key, invalidKeyRune) != -1
|
||||
}
|
||||
|
||||
func invalidKey(key []byte) bool {
|
||||
return len(key) == 0 || bytes.IndexFunc(key, invalidKeyRune) != -1
|
||||
// keyRuneFilter returns r for all valid key runes, and -1 for all invalid key
|
||||
// runes. When used as the mapping function for strings.Map and bytes.Map
|
||||
// functions it causes them to remove invalid key runes from strings or byte
|
||||
// slices respectively.
|
||||
func keyRuneFilter(r rune) rune {
|
||||
if r <= ' ' || r == '=' || r == '"' || r == utf8.RuneError {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func writeStringKey(w io.Writer, key string) error {
|
||||
if invalidKeyString(key) {
|
||||
k := strings.Map(keyRuneFilter, key)
|
||||
if k == "" {
|
||||
return ErrInvalidKey
|
||||
}
|
||||
_, err := io.WriteString(w, key)
|
||||
_, err := io.WriteString(w, k)
|
||||
return err
|
||||
}
|
||||
|
||||
func writeBytesKey(w io.Writer, key []byte) error {
|
||||
if invalidKey(key) {
|
||||
k := bytes.Map(keyRuneFilter, key)
|
||||
if len(k) == 0 {
|
||||
return ErrInvalidKey
|
||||
}
|
||||
_, err := w.Write(key)
|
||||
_, err := w.Write(k)
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -278,7 +279,7 @@ func safeError(err error) (s string, ok bool) {
|
|||
if v := reflect.ValueOf(err); v.Kind() == reflect.Ptr && v.IsNil() {
|
||||
s, ok = "null", false
|
||||
} else {
|
||||
panic(panicVal)
|
||||
s, ok = fmt.Sprintf("PANIC:%v", panicVal), false
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
@ -292,7 +293,7 @@ func safeString(str fmt.Stringer) (s string, ok bool) {
|
|||
if v := reflect.ValueOf(str); v.Kind() == reflect.Ptr && v.IsNil() {
|
||||
s, ok = "null", false
|
||||
} else {
|
||||
panic(panicVal)
|
||||
s, ok = fmt.Sprintf("PANIC:%v", panicVal), true
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
@ -306,7 +307,7 @@ func safeMarshal(tm encoding.TextMarshaler) (b []byte, err error) {
|
|||
if v := reflect.ValueOf(tm); v.Kind() == reflect.Ptr && v.IsNil() {
|
||||
b, err = nil, nil
|
||||
} else {
|
||||
panic(panicVal)
|
||||
b, err = nil, fmt.Errorf("panic when marshalling: %s", panicVal)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
|
3
vendor/github.com/go-logfmt/logfmt/go.mod
generated
vendored
Normal file
3
vendor/github.com/go-logfmt/logfmt/go.mod
generated
vendored
Normal file
|
@ -0,0 +1,3 @@
|
|||
module github.com/go-logfmt/logfmt
|
||||
|
||||
require github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515
|
2
vendor/github.com/go-logfmt/logfmt/go.sum
generated
vendored
Normal file
2
vendor/github.com/go-logfmt/logfmt/go.sum
generated
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
1
vendor/github.com/hashicorp/go-version/.travis.yml
generated
vendored
1
vendor/github.com/hashicorp/go-version/.travis.yml
generated
vendored
|
@ -7,6 +7,7 @@ go:
|
|||
- 1.3
|
||||
- 1.4
|
||||
- 1.9
|
||||
- "1.10"
|
||||
|
||||
script:
|
||||
- go test
|
||||
|
|
34
vendor/github.com/hashicorp/go-version/constraint.go
generated
vendored
34
vendor/github.com/hashicorp/go-version/constraint.go
generated
vendored
|
@ -2,6 +2,7 @@ package version
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
@ -113,6 +114,26 @@ func parseSingle(v string) (*Constraint, error) {
|
|||
}, nil
|
||||
}
|
||||
|
||||
func prereleaseCheck(v, c *Version) bool {
|
||||
switch vPre, cPre := v.Prerelease() != "", c.Prerelease() != ""; {
|
||||
case cPre && vPre:
|
||||
// A constraint with a pre-release can only match a pre-release version
|
||||
// with the same base segments.
|
||||
return reflect.DeepEqual(c.Segments64(), v.Segments64())
|
||||
|
||||
case !cPre && vPre:
|
||||
// A constraint without a pre-release can only match a version without a
|
||||
// pre-release.
|
||||
return false
|
||||
|
||||
case cPre && !vPre:
|
||||
// OK, except with the pessimistic operator
|
||||
case !cPre && !vPre:
|
||||
// OK
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Constraint functions
|
||||
//-------------------------------------------------------------------
|
||||
|
@ -126,22 +147,27 @@ func constraintNotEqual(v, c *Version) bool {
|
|||
}
|
||||
|
||||
func constraintGreaterThan(v, c *Version) bool {
|
||||
return v.Compare(c) == 1
|
||||
return prereleaseCheck(v, c) && v.Compare(c) == 1
|
||||
}
|
||||
|
||||
func constraintLessThan(v, c *Version) bool {
|
||||
return v.Compare(c) == -1
|
||||
return prereleaseCheck(v, c) && v.Compare(c) == -1
|
||||
}
|
||||
|
||||
func constraintGreaterThanEqual(v, c *Version) bool {
|
||||
return v.Compare(c) >= 0
|
||||
return prereleaseCheck(v, c) && v.Compare(c) >= 0
|
||||
}
|
||||
|
||||
func constraintLessThanEqual(v, c *Version) bool {
|
||||
return v.Compare(c) <= 0
|
||||
return prereleaseCheck(v, c) && v.Compare(c) <= 0
|
||||
}
|
||||
|
||||
func constraintPessimistic(v, c *Version) bool {
|
||||
// Using a pessimistic constraint with a pre-release, restricts versions to pre-releases
|
||||
if !prereleaseCheck(v, c) || (c.Prerelease() != "" && v.Prerelease() == "") {
|
||||
return false
|
||||
}
|
||||
|
||||
// If the version being checked is naturally less than the constraint, then there
|
||||
// is no way for the version to be valid against the constraint
|
||||
if v.LessThan(c) {
|
||||
|
|
1
vendor/github.com/hashicorp/go-version/go.mod
generated
vendored
Normal file
1
vendor/github.com/hashicorp/go-version/go.mod
generated
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
module github.com/hashicorp/go-version
|
62
vendor/github.com/hashicorp/go-version/version.go
generated
vendored
62
vendor/github.com/hashicorp/go-version/version.go
generated
vendored
|
@ -10,14 +10,25 @@ import (
|
|||
)
|
||||
|
||||
// The compiled regular expression used to test the validity of a version.
|
||||
var versionRegexp *regexp.Regexp
|
||||
var (
|
||||
versionRegexp *regexp.Regexp
|
||||
semverRegexp *regexp.Regexp
|
||||
)
|
||||
|
||||
// The raw regular expression string used for testing the validity
|
||||
// of a version.
|
||||
const VersionRegexpRaw string = `v?([0-9]+(\.[0-9]+)*?)` +
|
||||
`(-?([0-9A-Za-z\-~]+(\.[0-9A-Za-z\-~]+)*))?` +
|
||||
`(\+([0-9A-Za-z\-~]+(\.[0-9A-Za-z\-~]+)*))?` +
|
||||
`?`
|
||||
const (
|
||||
VersionRegexpRaw string = `v?([0-9]+(\.[0-9]+)*?)` +
|
||||
`(-([0-9]+[0-9A-Za-z\-~]*(\.[0-9A-Za-z\-~]+)*)|(-?([A-Za-z\-~]+[0-9A-Za-z\-~]*(\.[0-9A-Za-z\-~]+)*)))?` +
|
||||
`(\+([0-9A-Za-z\-~]+(\.[0-9A-Za-z\-~]+)*))?` +
|
||||
`?`
|
||||
|
||||
// SemverRegexpRaw requires a separator between version and prerelease
|
||||
SemverRegexpRaw string = `v?([0-9]+(\.[0-9]+)*?)` +
|
||||
`(-([0-9]+[0-9A-Za-z\-~]*(\.[0-9A-Za-z\-~]+)*)|(-([A-Za-z\-~]+[0-9A-Za-z\-~]*(\.[0-9A-Za-z\-~]+)*)))?` +
|
||||
`(\+([0-9A-Za-z\-~]+(\.[0-9A-Za-z\-~]+)*))?` +
|
||||
`?`
|
||||
)
|
||||
|
||||
// Version represents a single version.
|
||||
type Version struct {
|
||||
|
@ -25,16 +36,29 @@ type Version struct {
|
|||
pre string
|
||||
segments []int64
|
||||
si int
|
||||
original string
|
||||
}
|
||||
|
||||
func init() {
|
||||
versionRegexp = regexp.MustCompile("^" + VersionRegexpRaw + "$")
|
||||
semverRegexp = regexp.MustCompile("^" + SemverRegexpRaw + "$")
|
||||
}
|
||||
|
||||
// NewVersion parses the given version and returns a new
|
||||
// Version.
|
||||
func NewVersion(v string) (*Version, error) {
|
||||
matches := versionRegexp.FindStringSubmatch(v)
|
||||
return newVersion(v, versionRegexp)
|
||||
}
|
||||
|
||||
// NewSemver parses the given version and returns a new
|
||||
// Version that adheres strictly to SemVer specs
|
||||
// https://semver.org/
|
||||
func NewSemver(v string) (*Version, error) {
|
||||
return newVersion(v, semverRegexp)
|
||||
}
|
||||
|
||||
func newVersion(v string, pattern *regexp.Regexp) (*Version, error) {
|
||||
matches := pattern.FindStringSubmatch(v)
|
||||
if matches == nil {
|
||||
return nil, fmt.Errorf("Malformed version: %s", v)
|
||||
}
|
||||
|
@ -59,11 +83,17 @@ func NewVersion(v string) (*Version, error) {
|
|||
segments = append(segments, 0)
|
||||
}
|
||||
|
||||
pre := matches[7]
|
||||
if pre == "" {
|
||||
pre = matches[4]
|
||||
}
|
||||
|
||||
return &Version{
|
||||
metadata: matches[7],
|
||||
pre: matches[4],
|
||||
metadata: matches[10],
|
||||
pre: pre,
|
||||
segments: segments,
|
||||
si: si,
|
||||
original: v,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
@ -301,11 +331,19 @@ func (v *Version) Segments() []int {
|
|||
// for a version "1.2.3-beta", segments will return a slice of
|
||||
// 1, 2, 3.
|
||||
func (v *Version) Segments64() []int64 {
|
||||
return v.segments
|
||||
result := make([]int64, len(v.segments))
|
||||
copy(result, v.segments)
|
||||
return result
|
||||
}
|
||||
|
||||
// String returns the full version string included pre-release
|
||||
// and metadata information.
|
||||
//
|
||||
// This value is rebuilt according to the parsed segments and other
|
||||
// information. Therefore, ambiguities in the version string such as
|
||||
// prefixed zeroes (1.04.0 => 1.4.0), `v` prefix (v1.0.0 => 1.0.0), and
|
||||
// missing parts (1.0 => 1.0.0) will be made into a canonicalized form
|
||||
// as shown in the parenthesized examples.
|
||||
func (v *Version) String() string {
|
||||
var buf bytes.Buffer
|
||||
fmtParts := make([]string, len(v.segments))
|
||||
|
@ -324,3 +362,9 @@ func (v *Version) String() string {
|
|||
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// Original returns the original parsed version as-is, including any
|
||||
// potential whitespace, `v` prefix, etc.
|
||||
func (v *Version) Original() string {
|
||||
return v.original
|
||||
}
|
||||
|
|
30
vendor/github.com/hashicorp/golang-lru/lru.go
generated
vendored
30
vendor/github.com/hashicorp/golang-lru/lru.go
generated
vendored
|
@ -40,31 +40,35 @@ func (c *Cache) Purge() {
|
|||
// Add adds a value to the cache. Returns true if an eviction occurred.
|
||||
func (c *Cache) Add(key, value interface{}) (evicted bool) {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
return c.lru.Add(key, value)
|
||||
evicted = c.lru.Add(key, value)
|
||||
c.lock.Unlock()
|
||||
return evicted
|
||||
}
|
||||
|
||||
// Get looks up a key's value from the cache.
|
||||
func (c *Cache) Get(key interface{}) (value interface{}, ok bool) {
|
||||
c.lock.Lock()
|
||||
defer c.lock.Unlock()
|
||||
return c.lru.Get(key)
|
||||
value, ok = c.lru.Get(key)
|
||||
c.lock.Unlock()
|
||||
return value, ok
|
||||
}
|
||||
|
||||
// Contains checks if a key is in the cache, without updating the
|
||||
// recent-ness or deleting it for being stale.
|
||||
func (c *Cache) Contains(key interface{}) bool {
|
||||
c.lock.RLock()
|
||||
defer c.lock.RUnlock()
|
||||
return c.lru.Contains(key)
|
||||
containKey := c.lru.Contains(key)
|
||||
c.lock.RUnlock()
|
||||
return containKey
|
||||
}
|
||||
|
||||
// Peek returns the key value (or undefined if not found) without updating
|
||||
// the "recently used"-ness of the key.
|
||||
func (c *Cache) Peek(key interface{}) (value interface{}, ok bool) {
|
||||
c.lock.RLock()
|
||||
defer c.lock.RUnlock()
|
||||
return c.lru.Peek(key)
|
||||
value, ok = c.lru.Peek(key)
|
||||
c.lock.RUnlock()
|
||||
return value, ok
|
||||
}
|
||||
|
||||
// ContainsOrAdd checks if a key is in the cache without updating the
|
||||
|
@ -98,13 +102,15 @@ func (c *Cache) RemoveOldest() {
|
|||
// Keys returns a slice of the keys in the cache, from oldest to newest.
|
||||
func (c *Cache) Keys() []interface{} {
|
||||
c.lock.RLock()
|
||||
defer c.lock.RUnlock()
|
||||
return c.lru.Keys()
|
||||
keys := c.lru.Keys()
|
||||
c.lock.RUnlock()
|
||||
return keys
|
||||
}
|
||||
|
||||
// Len returns the number of items in the cache.
|
||||
func (c *Cache) Len() int {
|
||||
c.lock.RLock()
|
||||
defer c.lock.RUnlock()
|
||||
return c.lru.Len()
|
||||
length := c.lru.Len()
|
||||
c.lock.RUnlock()
|
||||
return length
|
||||
}
|
||||
|
|
78
vendor/github.com/improbable-eng/thanos/pkg/reloader/reloader.go
generated
vendored
78
vendor/github.com/improbable-eng/thanos/pkg/reloader/reloader.go
generated
vendored
|
@ -81,7 +81,7 @@ type Reloader struct {
|
|||
cfgFile string
|
||||
cfgOutputFile string
|
||||
ruleDirs []string
|
||||
ruleInterval time.Duration
|
||||
watchInterval time.Duration
|
||||
retryInterval time.Duration
|
||||
|
||||
lastCfgHash []byte
|
||||
|
@ -105,55 +105,74 @@ func New(logger log.Logger, reloadURL *url.URL, cfgFile string, cfgOutputFile st
|
|||
cfgFile: cfgFile,
|
||||
cfgOutputFile: cfgOutputFile,
|
||||
ruleDirs: ruleDirs,
|
||||
ruleInterval: 3 * time.Minute,
|
||||
watchInterval: 3 * time.Minute,
|
||||
retryInterval: 5 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// Watch starts to watch the config file and rules and process them until the context
|
||||
// We cannot detect everything via watch. Watch interval controls how often we re-read given dirs non-recursively.
|
||||
func (r *Reloader) WithWatchInterval(duration time.Duration) {
|
||||
r.watchInterval = duration
|
||||
}
|
||||
|
||||
// Watch starts to watch periodically the config file and rules and process them until the context
|
||||
// gets canceled. Config file gets env expanded if cfgOutputFile is specified and reload is trigger if
|
||||
// config or rules changed.
|
||||
// Watch watchers periodically based on r.watchInterval.
|
||||
// For config file it watches it directly as well via fsnotify.
|
||||
// It watches rule dirs as well, but lot's of edge cases are missing, so rely on interval mostly.
|
||||
func (r *Reloader) Watch(ctx context.Context) error {
|
||||
configWatcher, err := fsnotify.NewWatcher()
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create watcher")
|
||||
}
|
||||
defer runutil.CloseWithLogOnErr(r.logger, configWatcher, "config watcher close")
|
||||
defer runutil.CloseWithLogOnErr(r.logger, watcher, "config watcher close")
|
||||
|
||||
watchables := map[string]struct{}{}
|
||||
if r.cfgFile != "" {
|
||||
if err := configWatcher.Add(r.cfgFile); err != nil {
|
||||
return errors.Wrap(err, "add config file watch")
|
||||
watchables[filepath.Dir(r.cfgFile)] = struct{}{}
|
||||
if err := watcher.Add(r.cfgFile); err != nil {
|
||||
return errors.Wrapf(err, "add config file %s to watcher", r.cfgFile)
|
||||
}
|
||||
level.Info(r.logger).Log(
|
||||
"msg", "started watching config file for changes",
|
||||
"in", r.cfgFile,
|
||||
"out", r.cfgOutputFile)
|
||||
|
||||
err := r.apply(ctx)
|
||||
if err != nil {
|
||||
if err := r.apply(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
tick := time.NewTicker(r.ruleInterval)
|
||||
// Watch rule dirs in best effort manner.
|
||||
for _, ruleDir := range r.ruleDirs {
|
||||
watchables[filepath.Dir(ruleDir)] = struct{}{}
|
||||
if err := watcher.Add(ruleDir); err != nil {
|
||||
return errors.Wrapf(err, "add rule dir %s to watcher", ruleDir)
|
||||
}
|
||||
}
|
||||
|
||||
tick := time.NewTicker(r.watchInterval)
|
||||
defer tick.Stop()
|
||||
|
||||
level.Info(r.logger).Log(
|
||||
"msg", "started watching config file and non-recursively rule dirs for changes",
|
||||
"cfg", r.cfgFile,
|
||||
"out", r.cfgOutputFile,
|
||||
"dirs", strings.Join(r.ruleDirs, ","))
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-tick.C:
|
||||
case event := <-configWatcher.Events:
|
||||
if event.Name != r.cfgFile {
|
||||
case event := <-watcher.Events:
|
||||
// TODO(bwplotka): Add metric if we are not cycling CPU here too much.
|
||||
if _, ok := watchables[filepath.Dir(event.Name)]; !ok {
|
||||
continue
|
||||
}
|
||||
case err := <-configWatcher.Errors:
|
||||
case err := <-watcher.Errors:
|
||||
level.Error(r.logger).Log("msg", "watch error", "err", err)
|
||||
continue
|
||||
}
|
||||
|
||||
err := r.apply(ctx)
|
||||
if err != nil {
|
||||
if err := r.apply(ctx); err != nil {
|
||||
// Critical error.
|
||||
return err
|
||||
}
|
||||
|
@ -162,7 +181,7 @@ func (r *Reloader) Watch(ctx context.Context) error {
|
|||
|
||||
// apply triggers Prometheus reload if rules or config changed. If cfgOutputFile is set, we also
|
||||
// expand env vars into config file before reloading.
|
||||
// Reload is retried in retryInterval until ruleInterval.
|
||||
// Reload is retried in retryInterval until watchInterval.
|
||||
func (r *Reloader) apply(ctx context.Context) error {
|
||||
var (
|
||||
cfgHash []byte
|
||||
|
@ -199,9 +218,16 @@ func (r *Reloader) apply(ctx context.Context) error {
|
|||
return errors.Wrap(err, "expand environment variables")
|
||||
}
|
||||
|
||||
if err := ioutil.WriteFile(r.cfgOutputFile, b, 0666); err != nil {
|
||||
tmpFile := r.cfgOutputFile + ".tmp"
|
||||
defer func() {
|
||||
_ = os.Remove(tmpFile)
|
||||
}()
|
||||
if err := ioutil.WriteFile(tmpFile, b, 0666); err != nil {
|
||||
return errors.Wrap(err, "write file")
|
||||
}
|
||||
if err := os.Rename(tmpFile, r.cfgOutputFile); err != nil {
|
||||
return errors.Wrap(err, "rename file")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -247,8 +273,10 @@ func (r *Reloader) apply(ctx context.Context) error {
|
|||
}
|
||||
|
||||
// Retry trigger reload until it succeeded or next tick is near.
|
||||
retryCtx, cancel := context.WithTimeout(ctx, r.ruleInterval)
|
||||
err := runutil.RetryWithLog(r.logger, r.retryInterval, retryCtx.Done(), func() error {
|
||||
retryCtx, cancel := context.WithTimeout(ctx, r.watchInterval)
|
||||
defer cancel()
|
||||
|
||||
if err := runutil.RetryWithLog(r.logger, r.retryInterval, retryCtx.Done(), func() error {
|
||||
if err := r.triggerReload(ctx); err != nil {
|
||||
return errors.Wrap(err, "trigger reload")
|
||||
}
|
||||
|
@ -261,9 +289,7 @@ func (r *Reloader) apply(ctx context.Context) error {
|
|||
"cfg_out", r.cfgOutputFile,
|
||||
"rule_dirs", strings.Join(r.ruleDirs, ", "))
|
||||
return nil
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
}); err != nil {
|
||||
level.Error(r.logger).Log("msg", "Failed to trigger reload. Retrying.", "err", err)
|
||||
}
|
||||
|
||||
|
|
30
vendor/github.com/improbable-eng/thanos/pkg/runutil/runutil.go
generated
vendored
30
vendor/github.com/improbable-eng/thanos/pkg/runutil/runutil.go
generated
vendored
|
@ -33,7 +33,7 @@
|
|||
// For capturing error, use CloseWithErrCapture:
|
||||
//
|
||||
// var err error
|
||||
// defer runutil.CloseWithErrCapture(logger, &err, closer, "log format message")
|
||||
// defer runutil.CloseWithErrCapture(&err, closer, "log format message")
|
||||
//
|
||||
// // ...
|
||||
//
|
||||
|
@ -49,6 +49,7 @@ import (
|
|||
"github.com/go-kit/kit/log"
|
||||
"github.com/go-kit/kit/log/level"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/tsdb"
|
||||
)
|
||||
|
||||
// Repeat executes f every interval seconds until stopc is closed.
|
||||
|
@ -107,26 +108,13 @@ func CloseWithLogOnErr(logger log.Logger, closer io.Closer, format string, a ...
|
|||
level.Warn(logger).Log("msg", "detected close error", "err", errors.Wrap(err, fmt.Sprintf(format, a...)))
|
||||
}
|
||||
|
||||
// CloseWithErrCapture runs function and on error tries to return error by argument.
|
||||
// If error is already there we assume that error has higher priority and we just log the function error.
|
||||
func CloseWithErrCapture(logger log.Logger, err *error, closer io.Closer, format string, a ...interface{}) {
|
||||
closeErr := closer.Close()
|
||||
if closeErr == nil {
|
||||
return
|
||||
}
|
||||
// CloseWithErrCapture runs function and on error return error by argument including the given error (usually
|
||||
// from caller function).
|
||||
func CloseWithErrCapture(err *error, closer io.Closer, format string, a ...interface{}) {
|
||||
merr := tsdb.MultiError{}
|
||||
|
||||
if *err == nil {
|
||||
err = &closeErr
|
||||
return
|
||||
}
|
||||
merr.Add(*err)
|
||||
merr.Add(errors.Wrapf(closer.Close(), format, a...))
|
||||
|
||||
// There is already an error, let's log this one.
|
||||
if logger == nil {
|
||||
logger = log.NewLogfmtLogger(os.Stderr)
|
||||
}
|
||||
|
||||
level.Warn(logger).Log(
|
||||
"msg", "detected best effort close error that was preempted from the more important one",
|
||||
"err", errors.Wrap(closeErr, fmt.Sprintf(format, a...)),
|
||||
)
|
||||
*err = merr.Err()
|
||||
}
|
||||
|
|
29
vendor/github.com/oklog/ulid/.gitignore
generated
vendored
Normal file
29
vendor/github.com/oklog/ulid/.gitignore
generated
vendored
Normal file
|
@ -0,0 +1,29 @@
|
|||
#### joe made this: http://goel.io/joe
|
||||
|
||||
#####=== Go ===#####
|
||||
|
||||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
|
||||
# Folders
|
||||
_obj
|
||||
_test
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
*.[568vq]
|
||||
[568vq].out
|
||||
|
||||
*.cgo1.go
|
||||
*.cgo2.c
|
||||
_cgo_defun.c
|
||||
_cgo_gotypes.go
|
||||
_cgo_export.*
|
||||
|
||||
_testmain.go
|
||||
|
||||
*.exe
|
||||
*.test
|
||||
*.prof
|
||||
|
16
vendor/github.com/oklog/ulid/.travis.yml
generated
vendored
Normal file
16
vendor/github.com/oklog/ulid/.travis.yml
generated
vendored
Normal file
|
@ -0,0 +1,16 @@
|
|||
language: go
|
||||
sudo: false
|
||||
go:
|
||||
- 1.10.x
|
||||
install:
|
||||
- go get -v github.com/golang/lint/golint
|
||||
- go get golang.org/x/tools/cmd/cover
|
||||
- go get github.com/mattn/goveralls
|
||||
- go get -d -t -v ./...
|
||||
- go build -v ./...
|
||||
script:
|
||||
- go vet ./...
|
||||
- $HOME/gopath/bin/golint .
|
||||
- go test -v -race ./...
|
||||
- go test -v -covermode=count -coverprofile=cov.out
|
||||
- $HOME/gopath/bin/goveralls -coverprofile=cov.out -service=travis-ci -repotoken "$COVERALLS_TOKEN" || true
|
2
vendor/github.com/oklog/ulid/AUTHORS.md
generated
vendored
Normal file
2
vendor/github.com/oklog/ulid/AUTHORS.md
generated
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
- Peter Bourgon (@peterbourgon)
|
||||
- Tomás Senart (@tsenart)
|
33
vendor/github.com/oklog/ulid/CHANGELOG.md
generated
vendored
Normal file
33
vendor/github.com/oklog/ulid/CHANGELOG.md
generated
vendored
Normal file
|
@ -0,0 +1,33 @@
|
|||
## 1.3.1 / 2018-10-02
|
||||
|
||||
* Use underlying entropy source for random increments in Monotonic (#32)
|
||||
|
||||
## 1.3.0 / 2018-09-29
|
||||
|
||||
* Monotonic entropy support (#31)
|
||||
|
||||
## 1.2.0 / 2018-09-09
|
||||
|
||||
* Add a function to convert Unix time in milliseconds back to time.Time (#30)
|
||||
|
||||
## 1.1.0 / 2018-08-15
|
||||
|
||||
* Ensure random part is always read from the entropy reader in full (#28)
|
||||
|
||||
## 1.0.0 / 2018-07-29
|
||||
|
||||
* Add ParseStrict and MustParseStrict functions (#26)
|
||||
* Enforce overflow checking when parsing (#20)
|
||||
|
||||
## 0.3.0 / 2017-01-03
|
||||
|
||||
* Implement ULID.Compare method
|
||||
|
||||
## 0.2.0 / 2016-12-13
|
||||
|
||||
* Remove year 2262 Timestamp bug. (#1)
|
||||
* Gracefully handle invalid encodings when parsing.
|
||||
|
||||
## 0.1.0 / 2016-12-06
|
||||
|
||||
* First ULID release
|
17
vendor/github.com/oklog/ulid/CONTRIBUTING.md
generated
vendored
Normal file
17
vendor/github.com/oklog/ulid/CONTRIBUTING.md
generated
vendored
Normal file
|
@ -0,0 +1,17 @@
|
|||
# Contributing
|
||||
|
||||
We use GitHub to manage reviews of pull requests.
|
||||
|
||||
* If you have a trivial fix or improvement, go ahead and create a pull
|
||||
request, addressing (with `@...`) one or more of the maintainers
|
||||
(see [AUTHORS.md](AUTHORS.md)) in the description of the pull request.
|
||||
|
||||
* If you plan to do something more involved, first propose your ideas
|
||||
in a Github issue. This will avoid unnecessary work and surely give
|
||||
you and us a good deal of inspiration.
|
||||
|
||||
* Relevant coding style guidelines are the [Go Code Review
|
||||
Comments](https://code.google.com/p/go-wiki/wiki/CodeReviewComments)
|
||||
and the _Formatting and style_ section of Peter Bourgon's [Go: Best
|
||||
Practices for Production
|
||||
Environments](http://peter.bourgon.org/go-in-production/#formatting-and-style).
|
15
vendor/github.com/oklog/ulid/Gopkg.lock
generated
vendored
Normal file
15
vendor/github.com/oklog/ulid/Gopkg.lock
generated
vendored
Normal file
|
@ -0,0 +1,15 @@
|
|||
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
|
||||
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/pborman/getopt"
|
||||
packages = ["v2"]
|
||||
revision = "7148bc3a4c3008adfcab60cbebfd0576018f330b"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
inputs-digest = "6779b05abd5cd429c5393641d2453005a3cb74a400d161b2b5c5d0ca2e10e116"
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
26
vendor/github.com/oklog/ulid/Gopkg.toml
generated
vendored
Normal file
26
vendor/github.com/oklog/ulid/Gopkg.toml
generated
vendored
Normal file
|
@ -0,0 +1,26 @@
|
|||
|
||||
# Gopkg.toml example
|
||||
#
|
||||
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
|
||||
# for detailed Gopkg.toml documentation.
|
||||
#
|
||||
# required = ["github.com/user/thing/cmd/thing"]
|
||||
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
|
||||
#
|
||||
# [[constraint]]
|
||||
# name = "github.com/user/project"
|
||||
# version = "1.0.0"
|
||||
#
|
||||
# [[constraint]]
|
||||
# name = "github.com/user/project2"
|
||||
# branch = "dev"
|
||||
# source = "github.com/myfork/project2"
|
||||
#
|
||||
# [[override]]
|
||||
# name = "github.com/x/y"
|
||||
# version = "2.4.0"
|
||||
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/pborman/getopt"
|
201
vendor/github.com/oklog/ulid/LICENSE
generated
vendored
Normal file
201
vendor/github.com/oklog/ulid/LICENSE
generated
vendored
Normal file
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
150
vendor/github.com/oklog/ulid/README.md
generated
vendored
Normal file
150
vendor/github.com/oklog/ulid/README.md
generated
vendored
Normal file
|
@ -0,0 +1,150 @@
|
|||
# Universally Unique Lexicographically Sortable Identifier
|
||||
|
||||

|
||||
[](http://travis-ci.org/oklog/ulid)
|
||||
[](https://goreportcard.com/report/oklog/ulid)
|
||||
[](https://coveralls.io/github/oklog/ulid?branch=master)
|
||||
[](https://godoc.org/github.com/oklog/ulid)
|
||||
[](https://raw.githubusercontent.com/oklog/ulid/master/LICENSE)
|
||||
|
||||
A Go port of [alizain/ulid](https://github.com/alizain/ulid) with binary format implemented.
|
||||
|
||||
## Background
|
||||
|
||||
A GUID/UUID can be suboptimal for many use-cases because:
|
||||
|
||||
- It isn't the most character efficient way of encoding 128 bits
|
||||
- UUID v1/v2 is impractical in many environments, as it requires access to a unique, stable MAC address
|
||||
- UUID v3/v5 requires a unique seed and produces randomly distributed IDs, which can cause fragmentation in many data structures
|
||||
- UUID v4 provides no other information than randomness which can cause fragmentation in many data structures
|
||||
|
||||
A ULID however:
|
||||
|
||||
- Is compatible with UUID/GUID's
|
||||
- 1.21e+24 unique ULIDs per millisecond (1,208,925,819,614,629,174,706,176 to be exact)
|
||||
- Lexicographically sortable
|
||||
- Canonically encoded as a 26 character string, as opposed to the 36 character UUID
|
||||
- Uses Crockford's base32 for better efficiency and readability (5 bits per character)
|
||||
- Case insensitive
|
||||
- No special characters (URL safe)
|
||||
- Monotonic sort order (correctly detects and handles the same millisecond)
|
||||
|
||||
## Install
|
||||
|
||||
```shell
|
||||
go get github.com/oklog/ulid
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
An ULID is constructed with a `time.Time` and an `io.Reader` entropy source.
|
||||
This design allows for greater flexibility in choosing your trade-offs.
|
||||
|
||||
Please note that `rand.Rand` from the `math` package is *not* safe for concurrent use.
|
||||
Instantiate one per long living go-routine or use a `sync.Pool` if you want to avoid the potential contention of a locked `rand.Source` as its been frequently observed in the package level functions.
|
||||
|
||||
|
||||
```go
|
||||
func ExampleULID() {
|
||||
t := time.Unix(1000000, 0)
|
||||
entropy := ulid.Monotonic(rand.New(rand.NewSource(t.UnixNano())), 0)
|
||||
fmt.Println(ulid.MustNew(ulid.Timestamp(t), entropy))
|
||||
// Output: 0000XSNJG0MQJHBF4QX1EFD6Y3
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
## Specification
|
||||
|
||||
Below is the current specification of ULID as implemented in this repository.
|
||||
|
||||
### Components
|
||||
|
||||
**Timestamp**
|
||||
- 48 bits
|
||||
- UNIX-time in milliseconds
|
||||
- Won't run out of space till the year 10895 AD
|
||||
|
||||
**Entropy**
|
||||
- 80 bits
|
||||
- User defined entropy source.
|
||||
- Monotonicity within the same millisecond with [`ulid.Monotonic`](https://godoc.org/github.com/oklog/ulid#Monotonic)
|
||||
|
||||
### Encoding
|
||||
|
||||
[Crockford's Base32](http://www.crockford.com/wrmg/base32.html) is used as shown.
|
||||
This alphabet excludes the letters I, L, O, and U to avoid confusion and abuse.
|
||||
|
||||
```
|
||||
0123456789ABCDEFGHJKMNPQRSTVWXYZ
|
||||
```
|
||||
|
||||
### Binary Layout and Byte Order
|
||||
|
||||
The components are encoded as 16 octets. Each component is encoded with the Most Significant Byte first (network byte order).
|
||||
|
||||
```
|
||||
0 1 2 3
|
||||
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| 32_bit_uint_time_high |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| 16_bit_uint_time_low | 16_bit_uint_random |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| 32_bit_uint_random |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| 32_bit_uint_random |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
```
|
||||
|
||||
### String Representation
|
||||
|
||||
```
|
||||
01AN4Z07BY 79KA1307SR9X4MV3
|
||||
|----------| |----------------|
|
||||
Timestamp Entropy
|
||||
10 chars 16 chars
|
||||
48bits 80bits
|
||||
base32 base32
|
||||
```
|
||||
|
||||
## Test
|
||||
|
||||
```shell
|
||||
go test ./...
|
||||
```
|
||||
|
||||
## Benchmarks
|
||||
|
||||
On a Intel Core i7 Ivy Bridge 2.7 GHz, MacOS 10.12.1 and Go 1.8.0beta1
|
||||
|
||||
```
|
||||
BenchmarkNew/WithCryptoEntropy-8 2000000 771 ns/op 20.73 MB/s 16 B/op 1 allocs/op
|
||||
BenchmarkNew/WithEntropy-8 20000000 65.8 ns/op 243.01 MB/s 16 B/op 1 allocs/op
|
||||
BenchmarkNew/WithoutEntropy-8 50000000 30.0 ns/op 534.06 MB/s 16 B/op 1 allocs/op
|
||||
BenchmarkMustNew/WithCryptoEntropy-8 2000000 781 ns/op 20.48 MB/s 16 B/op 1 allocs/op
|
||||
BenchmarkMustNew/WithEntropy-8 20000000 70.0 ns/op 228.51 MB/s 16 B/op 1 allocs/op
|
||||
BenchmarkMustNew/WithoutEntropy-8 50000000 34.6 ns/op 462.98 MB/s 16 B/op 1 allocs/op
|
||||
BenchmarkParse-8 50000000 30.0 ns/op 866.16 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkMustParse-8 50000000 35.2 ns/op 738.94 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkString-8 20000000 64.9 ns/op 246.40 MB/s 32 B/op 1 allocs/op
|
||||
BenchmarkMarshal/Text-8 20000000 55.8 ns/op 286.84 MB/s 32 B/op 1 allocs/op
|
||||
BenchmarkMarshal/TextTo-8 100000000 22.4 ns/op 714.91 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkMarshal/Binary-8 300000000 4.02 ns/op 3981.77 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkMarshal/BinaryTo-8 2000000000 1.18 ns/op 13551.75 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkUnmarshal/Text-8 100000000 20.5 ns/op 1265.27 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkUnmarshal/Binary-8 300000000 4.94 ns/op 3240.01 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkNow-8 100000000 15.1 ns/op 528.09 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkTimestamp-8 2000000000 0.29 ns/op 27271.59 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkTime-8 2000000000 0.58 ns/op 13717.80 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkSetTime-8 2000000000 0.89 ns/op 9023.95 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkEntropy-8 200000000 7.62 ns/op 1311.66 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkSetEntropy-8 2000000000 0.88 ns/op 11376.54 MB/s 0 B/op 0 allocs/op
|
||||
BenchmarkCompare-8 200000000 7.34 ns/op 4359.23 MB/s 0 B/op 0 allocs/op
|
||||
```
|
||||
|
||||
## Prior Art
|
||||
|
||||
- [alizain/ulid](https://github.com/alizain/ulid)
|
||||
- [RobThree/NUlid](https://github.com/RobThree/NUlid)
|
||||
- [imdario/go-ulid](https://github.com/imdario/go-ulid)
|
614
vendor/github.com/oklog/ulid/ulid.go
generated
vendored
Normal file
614
vendor/github.com/oklog/ulid/ulid.go
generated
vendored
Normal file
|
@ -0,0 +1,614 @@
|
|||
// Copyright 2016 The Oklog Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ulid
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"database/sql/driver"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"math/bits"
|
||||
"math/rand"
|
||||
"time"
|
||||
)
|
||||
|
||||
/*
|
||||
An ULID is a 16 byte Universally Unique Lexicographically Sortable Identifier
|
||||
|
||||
The components are encoded as 16 octets.
|
||||
Each component is encoded with the MSB first (network byte order).
|
||||
|
||||
0 1 2 3
|
||||
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| 32_bit_uint_time_high |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| 16_bit_uint_time_low | 16_bit_uint_random |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| 32_bit_uint_random |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
| 32_bit_uint_random |
|
||||
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
*/
|
||||
type ULID [16]byte
|
||||
|
||||
var (
|
||||
// ErrDataSize is returned when parsing or unmarshaling ULIDs with the wrong
|
||||
// data size.
|
||||
ErrDataSize = errors.New("ulid: bad data size when unmarshaling")
|
||||
|
||||
// ErrInvalidCharacters is returned when parsing or unmarshaling ULIDs with
|
||||
// invalid Base32 encodings.
|
||||
ErrInvalidCharacters = errors.New("ulid: bad data characters when unmarshaling")
|
||||
|
||||
// ErrBufferSize is returned when marshalling ULIDs to a buffer of insufficient
|
||||
// size.
|
||||
ErrBufferSize = errors.New("ulid: bad buffer size when marshaling")
|
||||
|
||||
// ErrBigTime is returned when constructing an ULID with a time that is larger
|
||||
// than MaxTime.
|
||||
ErrBigTime = errors.New("ulid: time too big")
|
||||
|
||||
// ErrOverflow is returned when unmarshaling a ULID whose first character is
|
||||
// larger than 7, thereby exceeding the valid bit depth of 128.
|
||||
ErrOverflow = errors.New("ulid: overflow when unmarshaling")
|
||||
|
||||
// ErrMonotonicOverflow is returned by a Monotonic entropy source when
|
||||
// incrementing the previous ULID's entropy bytes would result in overflow.
|
||||
ErrMonotonicOverflow = errors.New("ulid: monotonic entropy overflow")
|
||||
|
||||
// ErrScanValue is returned when the value passed to scan cannot be unmarshaled
|
||||
// into the ULID.
|
||||
ErrScanValue = errors.New("ulid: source value must be a string or byte slice")
|
||||
)
|
||||
|
||||
// New returns an ULID with the given Unix milliseconds timestamp and an
|
||||
// optional entropy source. Use the Timestamp function to convert
|
||||
// a time.Time to Unix milliseconds.
|
||||
//
|
||||
// ErrBigTime is returned when passing a timestamp bigger than MaxTime.
|
||||
// Reading from the entropy source may also return an error.
|
||||
func New(ms uint64, entropy io.Reader) (id ULID, err error) {
|
||||
if err = id.SetTime(ms); err != nil {
|
||||
return id, err
|
||||
}
|
||||
|
||||
switch e := entropy.(type) {
|
||||
case nil:
|
||||
return id, err
|
||||
case *monotonic:
|
||||
err = e.MonotonicRead(ms, id[6:])
|
||||
default:
|
||||
_, err = io.ReadFull(e, id[6:])
|
||||
}
|
||||
|
||||
return id, err
|
||||
}
|
||||
|
||||
// MustNew is a convenience function equivalent to New that panics on failure
|
||||
// instead of returning an error.
|
||||
func MustNew(ms uint64, entropy io.Reader) ULID {
|
||||
id, err := New(ms, entropy)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// Parse parses an encoded ULID, returning an error in case of failure.
|
||||
//
|
||||
// ErrDataSize is returned if the len(ulid) is different from an encoded
|
||||
// ULID's length. Invalid encodings produce undefined ULIDs. For a version that
|
||||
// returns an error instead, see ParseStrict.
|
||||
func Parse(ulid string) (id ULID, err error) {
|
||||
return id, parse([]byte(ulid), false, &id)
|
||||
}
|
||||
|
||||
// ParseStrict parses an encoded ULID, returning an error in case of failure.
|
||||
//
|
||||
// It is like Parse, but additionally validates that the parsed ULID consists
|
||||
// only of valid base32 characters. It is slightly slower than Parse.
|
||||
//
|
||||
// ErrDataSize is returned if the len(ulid) is different from an encoded
|
||||
// ULID's length. Invalid encodings return ErrInvalidCharacters.
|
||||
func ParseStrict(ulid string) (id ULID, err error) {
|
||||
return id, parse([]byte(ulid), true, &id)
|
||||
}
|
||||
|
||||
func parse(v []byte, strict bool, id *ULID) error {
|
||||
// Check if a base32 encoded ULID is the right length.
|
||||
if len(v) != EncodedSize {
|
||||
return ErrDataSize
|
||||
}
|
||||
|
||||
// Check if all the characters in a base32 encoded ULID are part of the
|
||||
// expected base32 character set.
|
||||
if strict &&
|
||||
(dec[v[0]] == 0xFF ||
|
||||
dec[v[1]] == 0xFF ||
|
||||
dec[v[2]] == 0xFF ||
|
||||
dec[v[3]] == 0xFF ||
|
||||
dec[v[4]] == 0xFF ||
|
||||
dec[v[5]] == 0xFF ||
|
||||
dec[v[6]] == 0xFF ||
|
||||
dec[v[7]] == 0xFF ||
|
||||
dec[v[8]] == 0xFF ||
|
||||
dec[v[9]] == 0xFF ||
|
||||
dec[v[10]] == 0xFF ||
|
||||
dec[v[11]] == 0xFF ||
|
||||
dec[v[12]] == 0xFF ||
|
||||
dec[v[13]] == 0xFF ||
|
||||
dec[v[14]] == 0xFF ||
|
||||
dec[v[15]] == 0xFF ||
|
||||
dec[v[16]] == 0xFF ||
|
||||
dec[v[17]] == 0xFF ||
|
||||
dec[v[18]] == 0xFF ||
|
||||
dec[v[19]] == 0xFF ||
|
||||
dec[v[20]] == 0xFF ||
|
||||
dec[v[21]] == 0xFF ||
|
||||
dec[v[22]] == 0xFF ||
|
||||
dec[v[23]] == 0xFF ||
|
||||
dec[v[24]] == 0xFF ||
|
||||
dec[v[25]] == 0xFF) {
|
||||
return ErrInvalidCharacters
|
||||
}
|
||||
|
||||
// Check if the first character in a base32 encoded ULID will overflow. This
|
||||
// happens because the base32 representation encodes 130 bits, while the
|
||||
// ULID is only 128 bits.
|
||||
//
|
||||
// See https://github.com/oklog/ulid/issues/9 for details.
|
||||
if v[0] > '7' {
|
||||
return ErrOverflow
|
||||
}
|
||||
|
||||
// Use an optimized unrolled loop (from https://github.com/RobThree/NUlid)
|
||||
// to decode a base32 ULID.
|
||||
|
||||
// 6 bytes timestamp (48 bits)
|
||||
(*id)[0] = ((dec[v[0]] << 5) | dec[v[1]])
|
||||
(*id)[1] = ((dec[v[2]] << 3) | (dec[v[3]] >> 2))
|
||||
(*id)[2] = ((dec[v[3]] << 6) | (dec[v[4]] << 1) | (dec[v[5]] >> 4))
|
||||
(*id)[3] = ((dec[v[5]] << 4) | (dec[v[6]] >> 1))
|
||||
(*id)[4] = ((dec[v[6]] << 7) | (dec[v[7]] << 2) | (dec[v[8]] >> 3))
|
||||
(*id)[5] = ((dec[v[8]] << 5) | dec[v[9]])
|
||||
|
||||
// 10 bytes of entropy (80 bits)
|
||||
(*id)[6] = ((dec[v[10]] << 3) | (dec[v[11]] >> 2))
|
||||
(*id)[7] = ((dec[v[11]] << 6) | (dec[v[12]] << 1) | (dec[v[13]] >> 4))
|
||||
(*id)[8] = ((dec[v[13]] << 4) | (dec[v[14]] >> 1))
|
||||
(*id)[9] = ((dec[v[14]] << 7) | (dec[v[15]] << 2) | (dec[v[16]] >> 3))
|
||||
(*id)[10] = ((dec[v[16]] << 5) | dec[v[17]])
|
||||
(*id)[11] = ((dec[v[18]] << 3) | dec[v[19]]>>2)
|
||||
(*id)[12] = ((dec[v[19]] << 6) | (dec[v[20]] << 1) | (dec[v[21]] >> 4))
|
||||
(*id)[13] = ((dec[v[21]] << 4) | (dec[v[22]] >> 1))
|
||||
(*id)[14] = ((dec[v[22]] << 7) | (dec[v[23]] << 2) | (dec[v[24]] >> 3))
|
||||
(*id)[15] = ((dec[v[24]] << 5) | dec[v[25]])
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustParse is a convenience function equivalent to Parse that panics on failure
|
||||
// instead of returning an error.
|
||||
func MustParse(ulid string) ULID {
|
||||
id, err := Parse(ulid)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// MustParseStrict is a convenience function equivalent to ParseStrict that
|
||||
// panics on failure instead of returning an error.
|
||||
func MustParseStrict(ulid string) ULID {
|
||||
id, err := ParseStrict(ulid)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// String returns a lexicographically sortable string encoded ULID
|
||||
// (26 characters, non-standard base 32) e.g. 01AN4Z07BY79KA1307SR9X4MV3
|
||||
// Format: tttttttttteeeeeeeeeeeeeeee where t is time and e is entropy
|
||||
func (id ULID) String() string {
|
||||
ulid := make([]byte, EncodedSize)
|
||||
_ = id.MarshalTextTo(ulid)
|
||||
return string(ulid)
|
||||
}
|
||||
|
||||
// MarshalBinary implements the encoding.BinaryMarshaler interface by
|
||||
// returning the ULID as a byte slice.
|
||||
func (id ULID) MarshalBinary() ([]byte, error) {
|
||||
ulid := make([]byte, len(id))
|
||||
return ulid, id.MarshalBinaryTo(ulid)
|
||||
}
|
||||
|
||||
// MarshalBinaryTo writes the binary encoding of the ULID to the given buffer.
|
||||
// ErrBufferSize is returned when the len(dst) != 16.
|
||||
func (id ULID) MarshalBinaryTo(dst []byte) error {
|
||||
if len(dst) != len(id) {
|
||||
return ErrBufferSize
|
||||
}
|
||||
|
||||
copy(dst, id[:])
|
||||
return nil
|
||||
}
|
||||
|
||||
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface by
|
||||
// copying the passed data and converting it to an ULID. ErrDataSize is
|
||||
// returned if the data length is different from ULID length.
|
||||
func (id *ULID) UnmarshalBinary(data []byte) error {
|
||||
if len(data) != len(*id) {
|
||||
return ErrDataSize
|
||||
}
|
||||
|
||||
copy((*id)[:], data)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Encoding is the base 32 encoding alphabet used in ULID strings.
|
||||
const Encoding = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
||||
|
||||
// MarshalText implements the encoding.TextMarshaler interface by
|
||||
// returning the string encoded ULID.
|
||||
func (id ULID) MarshalText() ([]byte, error) {
|
||||
ulid := make([]byte, EncodedSize)
|
||||
return ulid, id.MarshalTextTo(ulid)
|
||||
}
|
||||
|
||||
// MarshalTextTo writes the ULID as a string to the given buffer.
|
||||
// ErrBufferSize is returned when the len(dst) != 26.
|
||||
func (id ULID) MarshalTextTo(dst []byte) error {
|
||||
// Optimized unrolled loop ahead.
|
||||
// From https://github.com/RobThree/NUlid
|
||||
|
||||
if len(dst) != EncodedSize {
|
||||
return ErrBufferSize
|
||||
}
|
||||
|
||||
// 10 byte timestamp
|
||||
dst[0] = Encoding[(id[0]&224)>>5]
|
||||
dst[1] = Encoding[id[0]&31]
|
||||
dst[2] = Encoding[(id[1]&248)>>3]
|
||||
dst[3] = Encoding[((id[1]&7)<<2)|((id[2]&192)>>6)]
|
||||
dst[4] = Encoding[(id[2]&62)>>1]
|
||||
dst[5] = Encoding[((id[2]&1)<<4)|((id[3]&240)>>4)]
|
||||
dst[6] = Encoding[((id[3]&15)<<1)|((id[4]&128)>>7)]
|
||||
dst[7] = Encoding[(id[4]&124)>>2]
|
||||
dst[8] = Encoding[((id[4]&3)<<3)|((id[5]&224)>>5)]
|
||||
dst[9] = Encoding[id[5]&31]
|
||||
|
||||
// 16 bytes of entropy
|
||||
dst[10] = Encoding[(id[6]&248)>>3]
|
||||
dst[11] = Encoding[((id[6]&7)<<2)|((id[7]&192)>>6)]
|
||||
dst[12] = Encoding[(id[7]&62)>>1]
|
||||
dst[13] = Encoding[((id[7]&1)<<4)|((id[8]&240)>>4)]
|
||||
dst[14] = Encoding[((id[8]&15)<<1)|((id[9]&128)>>7)]
|
||||
dst[15] = Encoding[(id[9]&124)>>2]
|
||||
dst[16] = Encoding[((id[9]&3)<<3)|((id[10]&224)>>5)]
|
||||
dst[17] = Encoding[id[10]&31]
|
||||
dst[18] = Encoding[(id[11]&248)>>3]
|
||||
dst[19] = Encoding[((id[11]&7)<<2)|((id[12]&192)>>6)]
|
||||
dst[20] = Encoding[(id[12]&62)>>1]
|
||||
dst[21] = Encoding[((id[12]&1)<<4)|((id[13]&240)>>4)]
|
||||
dst[22] = Encoding[((id[13]&15)<<1)|((id[14]&128)>>7)]
|
||||
dst[23] = Encoding[(id[14]&124)>>2]
|
||||
dst[24] = Encoding[((id[14]&3)<<3)|((id[15]&224)>>5)]
|
||||
dst[25] = Encoding[id[15]&31]
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Byte to index table for O(1) lookups when unmarshaling.
|
||||
// We use 0xFF as sentinel value for invalid indexes.
|
||||
var dec = [...]byte{
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01,
|
||||
0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E,
|
||||
0x0F, 0x10, 0x11, 0xFF, 0x12, 0x13, 0xFF, 0x14, 0x15, 0xFF,
|
||||
0x16, 0x17, 0x18, 0x19, 0x1A, 0xFF, 0x1B, 0x1C, 0x1D, 0x1E,
|
||||
0x1F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0A, 0x0B, 0x0C,
|
||||
0x0D, 0x0E, 0x0F, 0x10, 0x11, 0xFF, 0x12, 0x13, 0xFF, 0x14,
|
||||
0x15, 0xFF, 0x16, 0x17, 0x18, 0x19, 0x1A, 0xFF, 0x1B, 0x1C,
|
||||
0x1D, 0x1E, 0x1F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
|
||||
}
|
||||
|
||||
// EncodedSize is the length of a text encoded ULID.
|
||||
const EncodedSize = 26
|
||||
|
||||
// UnmarshalText implements the encoding.TextUnmarshaler interface by
|
||||
// parsing the data as string encoded ULID.
|
||||
//
|
||||
// ErrDataSize is returned if the len(v) is different from an encoded
|
||||
// ULID's length. Invalid encodings produce undefined ULIDs.
|
||||
func (id *ULID) UnmarshalText(v []byte) error {
|
||||
return parse(v, false, id)
|
||||
}
|
||||
|
||||
// Time returns the Unix time in milliseconds encoded in the ULID.
|
||||
// Use the top level Time function to convert the returned value to
|
||||
// a time.Time.
|
||||
func (id ULID) Time() uint64 {
|
||||
return uint64(id[5]) | uint64(id[4])<<8 |
|
||||
uint64(id[3])<<16 | uint64(id[2])<<24 |
|
||||
uint64(id[1])<<32 | uint64(id[0])<<40
|
||||
}
|
||||
|
||||
// maxTime is the maximum Unix time in milliseconds that can be
|
||||
// represented in an ULID.
|
||||
var maxTime = ULID{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}.Time()
|
||||
|
||||
// MaxTime returns the maximum Unix time in milliseconds that
|
||||
// can be encoded in an ULID.
|
||||
func MaxTime() uint64 { return maxTime }
|
||||
|
||||
// Now is a convenience function that returns the current
|
||||
// UTC time in Unix milliseconds. Equivalent to:
|
||||
// Timestamp(time.Now().UTC())
|
||||
func Now() uint64 { return Timestamp(time.Now().UTC()) }
|
||||
|
||||
// Timestamp converts a time.Time to Unix milliseconds.
|
||||
//
|
||||
// Because of the way ULID stores time, times from the year
|
||||
// 10889 produces undefined results.
|
||||
func Timestamp(t time.Time) uint64 {
|
||||
return uint64(t.Unix())*1000 +
|
||||
uint64(t.Nanosecond()/int(time.Millisecond))
|
||||
}
|
||||
|
||||
// Time converts Unix milliseconds in the format
|
||||
// returned by the Timestamp function to a time.Time.
|
||||
func Time(ms uint64) time.Time {
|
||||
s := int64(ms / 1e3)
|
||||
ns := int64((ms % 1e3) * 1e6)
|
||||
return time.Unix(s, ns)
|
||||
}
|
||||
|
||||
// SetTime sets the time component of the ULID to the given Unix time
|
||||
// in milliseconds.
|
||||
func (id *ULID) SetTime(ms uint64) error {
|
||||
if ms > maxTime {
|
||||
return ErrBigTime
|
||||
}
|
||||
|
||||
(*id)[0] = byte(ms >> 40)
|
||||
(*id)[1] = byte(ms >> 32)
|
||||
(*id)[2] = byte(ms >> 24)
|
||||
(*id)[3] = byte(ms >> 16)
|
||||
(*id)[4] = byte(ms >> 8)
|
||||
(*id)[5] = byte(ms)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Entropy returns the entropy from the ULID.
|
||||
func (id ULID) Entropy() []byte {
|
||||
e := make([]byte, 10)
|
||||
copy(e, id[6:])
|
||||
return e
|
||||
}
|
||||
|
||||
// SetEntropy sets the ULID entropy to the passed byte slice.
|
||||
// ErrDataSize is returned if len(e) != 10.
|
||||
func (id *ULID) SetEntropy(e []byte) error {
|
||||
if len(e) != 10 {
|
||||
return ErrDataSize
|
||||
}
|
||||
|
||||
copy((*id)[6:], e)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Compare returns an integer comparing id and other lexicographically.
|
||||
// The result will be 0 if id==other, -1 if id < other, and +1 if id > other.
|
||||
func (id ULID) Compare(other ULID) int {
|
||||
return bytes.Compare(id[:], other[:])
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface. It supports scanning
|
||||
// a string or byte slice.
|
||||
func (id *ULID) Scan(src interface{}) error {
|
||||
switch x := src.(type) {
|
||||
case nil:
|
||||
return nil
|
||||
case string:
|
||||
return id.UnmarshalText([]byte(x))
|
||||
case []byte:
|
||||
return id.UnmarshalBinary(x)
|
||||
}
|
||||
|
||||
return ErrScanValue
|
||||
}
|
||||
|
||||
// Value implements the sql/driver.Valuer interface. This returns the value
|
||||
// represented as a byte slice. If instead a string is desirable, a wrapper
|
||||
// type can be created that calls String().
|
||||
//
|
||||
// // stringValuer wraps a ULID as a string-based driver.Valuer.
|
||||
// type stringValuer ULID
|
||||
//
|
||||
// func (id stringValuer) Value() (driver.Value, error) {
|
||||
// return ULID(id).String(), nil
|
||||
// }
|
||||
//
|
||||
// // Example usage.
|
||||
// db.Exec("...", stringValuer(id))
|
||||
func (id ULID) Value() (driver.Value, error) {
|
||||
return id.MarshalBinary()
|
||||
}
|
||||
|
||||
// Monotonic returns an entropy source that is guaranteed to yield
|
||||
// strictly increasing entropy bytes for the same ULID timestamp.
|
||||
// On conflicts, the previous ULID entropy is incremented with a
|
||||
// random number between 1 and `inc` (inclusive).
|
||||
//
|
||||
// The provided entropy source must actually yield random bytes or else
|
||||
// monotonic reads are not guaranteed to terminate, since there isn't
|
||||
// enough randomness to compute an increment number.
|
||||
//
|
||||
// When `inc == 0`, it'll be set to a secure default of `math.MaxUint32`.
|
||||
// The lower the value of `inc`, the easier the next ULID within the
|
||||
// same millisecond is to guess. If your code depends on ULIDs having
|
||||
// secure entropy bytes, then don't go under this default unless you know
|
||||
// what you're doing.
|
||||
//
|
||||
// The returned io.Reader isn't safe for concurrent use.
|
||||
func Monotonic(entropy io.Reader, inc uint64) io.Reader {
|
||||
m := monotonic{
|
||||
Reader: bufio.NewReader(entropy),
|
||||
inc: inc,
|
||||
}
|
||||
|
||||
if m.inc == 0 {
|
||||
m.inc = math.MaxUint32
|
||||
}
|
||||
|
||||
if rng, ok := entropy.(*rand.Rand); ok {
|
||||
m.rng = rng
|
||||
}
|
||||
|
||||
return &m
|
||||
}
|
||||
|
||||
type monotonic struct {
|
||||
io.Reader
|
||||
ms uint64
|
||||
inc uint64
|
||||
entropy uint80
|
||||
rand [8]byte
|
||||
rng *rand.Rand
|
||||
}
|
||||
|
||||
func (m *monotonic) MonotonicRead(ms uint64, entropy []byte) (err error) {
|
||||
if !m.entropy.IsZero() && m.ms == ms {
|
||||
err = m.increment()
|
||||
m.entropy.AppendTo(entropy)
|
||||
} else if _, err = io.ReadFull(m.Reader, entropy); err == nil {
|
||||
m.ms = ms
|
||||
m.entropy.SetBytes(entropy)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// increment the previous entropy number with a random number
|
||||
// of up to m.inc (inclusive).
|
||||
func (m *monotonic) increment() error {
|
||||
if inc, err := m.random(); err != nil {
|
||||
return err
|
||||
} else if m.entropy.Add(inc) {
|
||||
return ErrMonotonicOverflow
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// random returns a uniform random value in [1, m.inc), reading entropy
|
||||
// from m.Reader. When m.inc == 0 || m.inc == 1, it returns 1.
|
||||
// Adapted from: https://golang.org/pkg/crypto/rand/#Int
|
||||
func (m *monotonic) random() (inc uint64, err error) {
|
||||
if m.inc <= 1 {
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
// Fast path for using a underlying rand.Rand directly.
|
||||
if m.rng != nil {
|
||||
// Range: [1, m.inc)
|
||||
return 1 + uint64(m.rng.Int63n(int64(m.inc))), nil
|
||||
}
|
||||
|
||||
// bitLen is the maximum bit length needed to encode a value < m.inc.
|
||||
bitLen := bits.Len64(m.inc)
|
||||
|
||||
// byteLen is the maximum byte length needed to encode a value < m.inc.
|
||||
byteLen := uint(bitLen+7) / 8
|
||||
|
||||
// msbitLen is the number of bits in the most significant byte of m.inc-1.
|
||||
msbitLen := uint(bitLen % 8)
|
||||
if msbitLen == 0 {
|
||||
msbitLen = 8
|
||||
}
|
||||
|
||||
for inc == 0 || inc >= m.inc {
|
||||
if _, err = io.ReadFull(m.Reader, m.rand[:byteLen]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Clear bits in the first byte to increase the probability
|
||||
// that the candidate is < m.inc.
|
||||
m.rand[0] &= uint8(int(1<<msbitLen) - 1)
|
||||
|
||||
// Convert the read bytes into an uint64 with byteLen
|
||||
// Optimized unrolled loop.
|
||||
switch byteLen {
|
||||
case 1:
|
||||
inc = uint64(m.rand[0])
|
||||
case 2:
|
||||
inc = uint64(binary.LittleEndian.Uint16(m.rand[:2]))
|
||||
case 3, 4:
|
||||
inc = uint64(binary.LittleEndian.Uint32(m.rand[:4]))
|
||||
case 5, 6, 7, 8:
|
||||
inc = uint64(binary.LittleEndian.Uint64(m.rand[:8]))
|
||||
}
|
||||
}
|
||||
|
||||
// Range: [1, m.inc)
|
||||
return 1 + inc, nil
|
||||
}
|
||||
|
||||
type uint80 struct {
|
||||
Hi uint16
|
||||
Lo uint64
|
||||
}
|
||||
|
||||
func (u *uint80) SetBytes(bs []byte) {
|
||||
u.Hi = binary.BigEndian.Uint16(bs[:2])
|
||||
u.Lo = binary.BigEndian.Uint64(bs[2:])
|
||||
}
|
||||
|
||||
func (u *uint80) AppendTo(bs []byte) {
|
||||
binary.BigEndian.PutUint16(bs[:2], u.Hi)
|
||||
binary.BigEndian.PutUint64(bs[2:], u.Lo)
|
||||
}
|
||||
|
||||
func (u *uint80) Add(n uint64) (overflow bool) {
|
||||
lo, hi := u.Lo, u.Hi
|
||||
if u.Lo += n; u.Lo < lo {
|
||||
u.Hi++
|
||||
}
|
||||
return u.Hi < hi
|
||||
}
|
||||
|
||||
func (u uint80) IsZero() bool {
|
||||
return u.Hi == 0 && u.Lo == 0
|
||||
}
|
4
vendor/github.com/prometheus/client_golang/prometheus/desc.go
generated
vendored
4
vendor/github.com/prometheus/client_golang/prometheus/desc.go
generated
vendored
|
@ -93,7 +93,7 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *
|
|||
// First add only the const label names and sort them...
|
||||
for labelName := range constLabels {
|
||||
if !checkLabelName(labelName) {
|
||||
d.err = fmt.Errorf("%q is not a valid label name", labelName)
|
||||
d.err = fmt.Errorf("%q is not a valid label name for metric %q", labelName, fqName)
|
||||
return d
|
||||
}
|
||||
labelNames = append(labelNames, labelName)
|
||||
|
@ -115,7 +115,7 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *
|
|||
// dimension with a different mix between preset and variable labels.
|
||||
for _, labelName := range variableLabels {
|
||||
if !checkLabelName(labelName) {
|
||||
d.err = fmt.Errorf("%q is not a valid label name", labelName)
|
||||
d.err = fmt.Errorf("%q is not a valid label name for metric %q", labelName, fqName)
|
||||
return d
|
||||
}
|
||||
labelNames = append(labelNames, "$"+labelName)
|
||||
|
|
223
vendor/github.com/prometheus/client_golang/prometheus/promauto/auto.go
generated
vendored
Normal file
223
vendor/github.com/prometheus/client_golang/prometheus/promauto/auto.go
generated
vendored
Normal file
|
@ -0,0 +1,223 @@
|
|||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package promauto provides constructors for the usual Prometheus metrics that
|
||||
// return them already registered with the global registry
|
||||
// (prometheus.DefaultRegisterer). This allows very compact code, avoiding any
|
||||
// references to the registry altogether, but all the constructors in this
|
||||
// package will panic if the registration fails.
|
||||
//
|
||||
// The following example is a complete program to create a histogram of normally
|
||||
// distributed random numbers from the math/rand package:
|
||||
//
|
||||
// package main
|
||||
//
|
||||
// import (
|
||||
// "math/rand"
|
||||
// "net/http"
|
||||
//
|
||||
// "github.com/prometheus/client_golang/prometheus"
|
||||
// "github.com/prometheus/client_golang/prometheus/promauto"
|
||||
// "github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
// )
|
||||
//
|
||||
// var histogram = promauto.NewHistogram(prometheus.HistogramOpts{
|
||||
// Name: "random_numbers",
|
||||
// Help: "A histogram of normally distributed random numbers.",
|
||||
// Buckets: prometheus.LinearBuckets(-3, .1, 61),
|
||||
// })
|
||||
//
|
||||
// func Random() {
|
||||
// for {
|
||||
// histogram.Observe(rand.NormFloat64())
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// func main() {
|
||||
// go Random()
|
||||
// http.Handle("/metrics", promhttp.Handler())
|
||||
// http.ListenAndServe(":1971", nil)
|
||||
// }
|
||||
//
|
||||
// Prometheus's version of a minimal hello-world program:
|
||||
//
|
||||
// package main
|
||||
//
|
||||
// import (
|
||||
// "fmt"
|
||||
// "net/http"
|
||||
//
|
||||
// "github.com/prometheus/client_golang/prometheus"
|
||||
// "github.com/prometheus/client_golang/prometheus/promauto"
|
||||
// "github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
// )
|
||||
//
|
||||
// func main() {
|
||||
// http.Handle("/", promhttp.InstrumentHandlerCounter(
|
||||
// promauto.NewCounterVec(
|
||||
// prometheus.CounterOpts{
|
||||
// Name: "hello_requests_total",
|
||||
// Help: "Total number of hello-world requests by HTTP code.",
|
||||
// },
|
||||
// []string{"code"},
|
||||
// ),
|
||||
// http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// fmt.Fprint(w, "Hello, world!")
|
||||
// }),
|
||||
// ))
|
||||
// http.Handle("/metrics", promhttp.Handler())
|
||||
// http.ListenAndServe(":1971", nil)
|
||||
// }
|
||||
//
|
||||
// This appears very handy. So why are these constructors locked away in a
|
||||
// separate package? There are two caveats:
|
||||
//
|
||||
// First, in more complex programs, global state is often quite problematic.
|
||||
// That's the reason why the metrics constructors in the prometheus package do
|
||||
// not interact with the global prometheus.DefaultRegisterer on their own. You
|
||||
// are free to use the Register or MustRegister functions to register them with
|
||||
// the global prometheus.DefaultRegisterer, but you could as well choose a local
|
||||
// Registerer (usually created with prometheus.NewRegistry, but there are other
|
||||
// scenarios, e.g. testing).
|
||||
//
|
||||
// The second issue is that registration may fail, e.g. if a metric inconsistent
|
||||
// with the newly to be registered one is already registered. But how to signal
|
||||
// and handle a panic in the automatic registration with the default registry?
|
||||
// The only way is panicking. While panicking on invalid input provided by the
|
||||
// programmer is certainly fine, things are a bit more subtle in this case: You
|
||||
// might just add another package to the program, and that package (in its init
|
||||
// function) happens to register a metric with the same name as your code. Now,
|
||||
// all of a sudden, either your code or the code of the newly imported package
|
||||
// panics, depending on initialization order, without any opportunity to handle
|
||||
// the case gracefully. Even worse is a scenario where registration happens
|
||||
// later during the runtime (e.g. upon loading some kind of plugin), where the
|
||||
// panic could be triggered long after the code has been deployed to
|
||||
// production. A possibility to panic should be explicitly called out by the
|
||||
// Must… idiom, cf. prometheus.MustRegister. But adding a separate set of
|
||||
// constructors in the prometheus package called MustRegisterNewCounterVec or
|
||||
// similar would be quite unwieldy. Adding an extra MustRegister method to each
|
||||
// metric, returning the registered metric, would result in nice code for those
|
||||
// using the method, but would pollute every single metric interface for
|
||||
// everybody avoiding the global registry.
|
||||
//
|
||||
// To address both issues, the problematic auto-registering and possibly
|
||||
// panicking constructors are all in this package with a clear warning
|
||||
// ahead. And whoever cares about avoiding global state and possibly panicking
|
||||
// function calls can simply ignore the existence of the promauto package
|
||||
// altogether.
|
||||
//
|
||||
// A final note: There is a similar case in the net/http package of the standard
|
||||
// library. It has DefaultServeMux as a global instance of ServeMux, and the
|
||||
// Handle function acts on it, panicking if a handler for the same pattern has
|
||||
// already been registered. However, one might argue that the whole HTTP routing
|
||||
// is usually set up closely together in the same package or file, while
|
||||
// Prometheus metrics tend to be spread widely over the codebase, increasing the
|
||||
// chance of surprising registration failures. Furthermore, the use of global
|
||||
// state in net/http has been criticized widely, and some avoid it altogether.
|
||||
package promauto
|
||||
|
||||
import "github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
// NewCounter works like the function of the same name in the prometheus package
|
||||
// but it automatically registers the Counter with the
|
||||
// prometheus.DefaultRegisterer. If the registration fails, NewCounter panics.
|
||||
func NewCounter(opts prometheus.CounterOpts) prometheus.Counter {
|
||||
c := prometheus.NewCounter(opts)
|
||||
prometheus.MustRegister(c)
|
||||
return c
|
||||
}
|
||||
|
||||
// NewCounterVec works like the function of the same name in the prometheus
|
||||
// package but it automatically registers the CounterVec with the
|
||||
// prometheus.DefaultRegisterer. If the registration fails, NewCounterVec
|
||||
// panics.
|
||||
func NewCounterVec(opts prometheus.CounterOpts, labelNames []string) *prometheus.CounterVec {
|
||||
c := prometheus.NewCounterVec(opts, labelNames)
|
||||
prometheus.MustRegister(c)
|
||||
return c
|
||||
}
|
||||
|
||||
// NewCounterFunc works like the function of the same name in the prometheus
|
||||
// package but it automatically registers the CounterFunc with the
|
||||
// prometheus.DefaultRegisterer. If the registration fails, NewCounterFunc
|
||||
// panics.
|
||||
func NewCounterFunc(opts prometheus.CounterOpts, function func() float64) prometheus.CounterFunc {
|
||||
g := prometheus.NewCounterFunc(opts, function)
|
||||
prometheus.MustRegister(g)
|
||||
return g
|
||||
}
|
||||
|
||||
// NewGauge works like the function of the same name in the prometheus package
|
||||
// but it automatically registers the Gauge with the
|
||||
// prometheus.DefaultRegisterer. If the registration fails, NewGauge panics.
|
||||
func NewGauge(opts prometheus.GaugeOpts) prometheus.Gauge {
|
||||
g := prometheus.NewGauge(opts)
|
||||
prometheus.MustRegister(g)
|
||||
return g
|
||||
}
|
||||
|
||||
// NewGaugeVec works like the function of the same name in the prometheus
|
||||
// package but it automatically registers the GaugeVec with the
|
||||
// prometheus.DefaultRegisterer. If the registration fails, NewGaugeVec panics.
|
||||
func NewGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *prometheus.GaugeVec {
|
||||
g := prometheus.NewGaugeVec(opts, labelNames)
|
||||
prometheus.MustRegister(g)
|
||||
return g
|
||||
}
|
||||
|
||||
// NewGaugeFunc works like the function of the same name in the prometheus
|
||||
// package but it automatically registers the GaugeFunc with the
|
||||
// prometheus.DefaultRegisterer. If the registration fails, NewGaugeFunc panics.
|
||||
func NewGaugeFunc(opts prometheus.GaugeOpts, function func() float64) prometheus.GaugeFunc {
|
||||
g := prometheus.NewGaugeFunc(opts, function)
|
||||
prometheus.MustRegister(g)
|
||||
return g
|
||||
}
|
||||
|
||||
// NewSummary works like the function of the same name in the prometheus package
|
||||
// but it automatically registers the Summary with the
|
||||
// prometheus.DefaultRegisterer. If the registration fails, NewSummary panics.
|
||||
func NewSummary(opts prometheus.SummaryOpts) prometheus.Summary {
|
||||
s := prometheus.NewSummary(opts)
|
||||
prometheus.MustRegister(s)
|
||||
return s
|
||||
}
|
||||
|
||||
// NewSummaryVec works like the function of the same name in the prometheus
|
||||
// package but it automatically registers the SummaryVec with the
|
||||
// prometheus.DefaultRegisterer. If the registration fails, NewSummaryVec
|
||||
// panics.
|
||||
func NewSummaryVec(opts prometheus.SummaryOpts, labelNames []string) *prometheus.SummaryVec {
|
||||
s := prometheus.NewSummaryVec(opts, labelNames)
|
||||
prometheus.MustRegister(s)
|
||||
return s
|
||||
}
|
||||
|
||||
// NewHistogram works like the function of the same name in the prometheus
|
||||
// package but it automatically registers the Histogram with the
|
||||
// prometheus.DefaultRegisterer. If the registration fails, NewHistogram panics.
|
||||
func NewHistogram(opts prometheus.HistogramOpts) prometheus.Histogram {
|
||||
h := prometheus.NewHistogram(opts)
|
||||
prometheus.MustRegister(h)
|
||||
return h
|
||||
}
|
||||
|
||||
// NewHistogramVec works like the function of the same name in the prometheus
|
||||
// package but it automatically registers the HistogramVec with the
|
||||
// prometheus.DefaultRegisterer. If the registration fails, NewHistogramVec
|
||||
// panics.
|
||||
func NewHistogramVec(opts prometheus.HistogramOpts, labelNames []string) *prometheus.HistogramVec {
|
||||
h := prometheus.NewHistogramVec(opts, labelNames)
|
||||
prometheus.MustRegister(h)
|
||||
return h
|
||||
}
|
12
vendor/github.com/prometheus/client_golang/prometheus/registry.go
generated
vendored
12
vendor/github.com/prometheus/client_golang/prometheus/registry.go
generated
vendored
|
@ -872,7 +872,13 @@ func checkMetricConsistency(
|
|||
h = hashAddByte(h, separatorByte)
|
||||
// Make sure label pairs are sorted. We depend on it for the consistency
|
||||
// check.
|
||||
sort.Sort(labelPairSorter(dtoMetric.Label))
|
||||
if !sort.IsSorted(labelPairSorter(dtoMetric.Label)) {
|
||||
// We cannot sort dtoMetric.Label in place as it is immutable by contract.
|
||||
copiedLabels := make([]*dto.LabelPair, len(dtoMetric.Label))
|
||||
copy(copiedLabels, dtoMetric.Label)
|
||||
sort.Sort(labelPairSorter(copiedLabels))
|
||||
dtoMetric.Label = copiedLabels
|
||||
}
|
||||
for _, lp := range dtoMetric.Label {
|
||||
h = hashAdd(h, lp.GetName())
|
||||
h = hashAddByte(h, separatorByte)
|
||||
|
@ -903,8 +909,8 @@ func checkDescConsistency(
|
|||
}
|
||||
|
||||
// Is the desc consistent with the content of the metric?
|
||||
lpsFromDesc := make([]*dto.LabelPair, 0, len(dtoMetric.Label))
|
||||
lpsFromDesc = append(lpsFromDesc, desc.constLabelPairs...)
|
||||
lpsFromDesc := make([]*dto.LabelPair, len(desc.constLabelPairs), len(dtoMetric.Label))
|
||||
copy(lpsFromDesc, desc.constLabelPairs)
|
||||
for _, l := range desc.variableLabels {
|
||||
lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{
|
||||
Name: proto.String(l),
|
||||
|
|
11
vendor/github.com/prometheus/client_golang/prometheus/timer.go
generated
vendored
11
vendor/github.com/prometheus/client_golang/prometheus/timer.go
generated
vendored
|
@ -39,13 +39,16 @@ func NewTimer(o Observer) *Timer {
|
|||
|
||||
// ObserveDuration records the duration passed since the Timer was created with
|
||||
// NewTimer. It calls the Observe method of the Observer provided during
|
||||
// construction with the duration in seconds as an argument. ObserveDuration is
|
||||
// usually called with a defer statement.
|
||||
// construction with the duration in seconds as an argument. The observed
|
||||
// duration is also returned. ObserveDuration is usually called with a defer
|
||||
// statement.
|
||||
//
|
||||
// Note that this method is only guaranteed to never observe negative durations
|
||||
// if used with Go1.9+.
|
||||
func (t *Timer) ObserveDuration() {
|
||||
func (t *Timer) ObserveDuration() time.Duration {
|
||||
d := time.Since(t.begin)
|
||||
if t.observer != nil {
|
||||
t.observer.Observe(time.Since(t.begin).Seconds())
|
||||
t.observer.Observe(d.Seconds())
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
|
1
vendor/github.com/prometheus/tsdb/.gitignore
generated
vendored
Normal file
1
vendor/github.com/prometheus/tsdb/.gitignore
generated
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
benchout/
|
22
vendor/github.com/prometheus/tsdb/.travis.yml
generated
vendored
Normal file
22
vendor/github.com/prometheus/tsdb/.travis.yml
generated
vendored
Normal file
|
@ -0,0 +1,22 @@
|
|||
dist: trusty
|
||||
language: go
|
||||
os:
|
||||
- windows
|
||||
- linux
|
||||
- osx
|
||||
|
||||
go:
|
||||
- 1.10.x
|
||||
- 1.11.x
|
||||
- 1.12.x
|
||||
|
||||
go_import_path: github.com/prometheus/tsdb
|
||||
|
||||
before_install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then choco install make; fi
|
||||
|
||||
install:
|
||||
- make deps
|
||||
|
||||
script:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then make test; else make all; fi
|
45
vendor/github.com/prometheus/tsdb/CHANGELOG.md
generated
vendored
Normal file
45
vendor/github.com/prometheus/tsdb/CHANGELOG.md
generated
vendored
Normal file
|
@ -0,0 +1,45 @@
|
|||
## master / unreleased
|
||||
|
||||
## 0.6.1
|
||||
- [BUGFIX] Update `last` after appending a non-overlapping chunk in `chunks.MergeOverlappingChunks`. [#539](https://github.com/prometheus/tsdb/pull/539)
|
||||
|
||||
## 0.6.0
|
||||
- [CHANGE] `AllowOverlappingBlock` is now `AllowOverlappingBlocks`.
|
||||
|
||||
## 0.5.0
|
||||
- [FEATURE] Time-ovelapping blocks are now allowed. [#370](https://github.com/prometheus/tsdb/pull/370)
|
||||
- Disabled by default and can be enabled via `AllowOverlappingBlock` option.
|
||||
- Added `MergeChunks` function in `chunkenc/xor.go` to merge 2 time-overlapping chunks.
|
||||
- Added `MergeOverlappingChunks` function in `chunks/chunks.go` to merge multiple time-overlapping Chunk Metas.
|
||||
- Added `MinTime` and `MaxTime` method for `BlockReader`.
|
||||
- [FEATURE] New `dump` command to tsdb tool to dump all samples.
|
||||
- [FEATURE] New `encoding` package for common binary encoding/decoding helpers.
|
||||
- Added to remove some code duplication.
|
||||
- [ENHANCEMENT] When closing the db any running compaction will be cancelled so it doesn't block.
|
||||
- `NewLeveledCompactor` takes a context.
|
||||
- [CHANGE] `prometheus_tsdb_storage_blocks_bytes_total` is now `prometheus_tsdb_storage_blocks_bytes`.
|
||||
- [BUGFIX] Improved Postings Merge performance. Fixes a regression from the the previous release.
|
||||
- [BUGFIX] LiveReader can get into an infinite loop on corrupt WALs.
|
||||
|
||||
## 0.4.0
|
||||
- [CHANGE] New `WALSegmentSize` option to override the `DefaultOptions.WALSegmentSize`. Added to allow using smaller wal files. For example using tmpfs on a RPI to minimise the SD card wear out from the constant WAL writes. As part of this change the `DefaultOptions.WALSegmentSize` constant was also exposed.
|
||||
- [CHANGE] Empty blocks are not written during compaction [#374](https://github.com/prometheus/tsdb/pull/374)
|
||||
- [FEATURE] Size base retention through `Options.MaxBytes`. As part of this change:
|
||||
- Added new metrics - `prometheus_tsdb_storage_blocks_bytes_total`, `prometheus_tsdb_size_retentions_total`, `prometheus_tsdb_time_retentions_total`
|
||||
- New public interface `SizeReader: Size() int64`
|
||||
- `OpenBlock` signature changed to take a logger.
|
||||
- [REMOVED] `PrefixMatcher` is considered unused so was removed.
|
||||
- [CLEANUP] `Options.WALFlushInterval` is removed as it wasn't used anywhere.
|
||||
- [FEATURE] Add new `LiveReader` to WAL pacakge. Added to allow live tailing of a WAL segment, used by Prometheus Remote Write after refactor. The main difference between the new reader and the existing `Reader` is that for `LiveReader` a call to `Next()` that returns false does not mean that there will never be more data to read.
|
||||
|
||||
## 0.3.1
|
||||
- [BUGFIX] Fixed most windows test and some actual bugs for unclosed file readers.
|
||||
|
||||
## 0.3.0
|
||||
- [CHANGE] `LastCheckpoint()` used to return just the segment name and now it returns the full relative path.
|
||||
- [CHANGE] `NewSegmentsRangeReader()` can now read over miltiple wal ranges by using the new `SegmentRange{}` struct.
|
||||
- [CHANGE] `CorruptionErr{}` now also exposes the Segment `Dir` which is added when displaying any errors.
|
||||
- [CHANGE] `Head.Init()` is changed to `Head.Init(minValidTime int64)`
|
||||
- [CHANGE] `SymbolTable()` renamed to `SymbolTableSize()` to make the name consistent with the `Block{ symbolTableSize uint64 }` field.
|
||||
- [CHANGE] `wal.Reader{}` now exposes `Segment()` for the current segment being read and `Offset()` for the current offset.
|
||||
- [FEATURE] tsdbutil analyze subcomand to find churn, high cardinality, etc.
|
201
vendor/github.com/prometheus/tsdb/LICENSE
generated
vendored
Normal file
201
vendor/github.com/prometheus/tsdb/LICENSE
generated
vendored
Normal file
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
4
vendor/github.com/prometheus/tsdb/MAINTAINERS.md
generated
vendored
Normal file
4
vendor/github.com/prometheus/tsdb/MAINTAINERS.md
generated
vendored
Normal file
|
@ -0,0 +1,4 @@
|
|||
Maintainers of this repository:
|
||||
|
||||
* Krasi Georgiev <kgeorgie@redhat.com> @krasi-georgiev
|
||||
* Goutham Veeramachaneni <gouthamve@gmail.com> @gouthamve
|
38
vendor/github.com/prometheus/tsdb/Makefile
generated
vendored
Normal file
38
vendor/github.com/prometheus/tsdb/Makefile
generated
vendored
Normal file
|
@ -0,0 +1,38 @@
|
|||
# Copyright 2018 The Prometheus Authors
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
TSDB_PROJECT_DIR = "."
|
||||
TSDB_CLI_DIR="$(TSDB_PROJECT_DIR)/cmd/tsdb"
|
||||
TSDB_BIN = "$(TSDB_CLI_DIR)/tsdb"
|
||||
TSDB_BENCHMARK_NUM_METRICS ?= 1000
|
||||
TSDB_BENCHMARK_DATASET ?= "$(TSDB_PROJECT_DIR)/testdata/20kseries.json"
|
||||
TSDB_BENCHMARK_OUTPUT_DIR ?= "$(TSDB_CLI_DIR)/benchout"
|
||||
|
||||
include Makefile.common
|
||||
|
||||
.PHONY: deps
|
||||
deps:
|
||||
@echo ">> getting dependencies"
|
||||
GO111MODULE=$(GO111MODULE) $(GO) get $(GOOPTS) -t ./...
|
||||
|
||||
build:
|
||||
GO111MODULE=$(GO111MODULE) $(GO) build -o $(TSDB_BIN) $(TSDB_CLI_DIR)
|
||||
|
||||
bench: build
|
||||
@echo ">> running benchmark, writing result to $(TSDB_BENCHMARK_OUTPUT_DIR)"
|
||||
@$(TSDB_BIN) bench write --metrics=$(TSDB_BENCHMARK_NUM_METRICS) --out=$(TSDB_BENCHMARK_OUTPUT_DIR) $(TSDB_BENCHMARK_DATASET)
|
||||
@$(GO) tool pprof -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/cpu.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/cpuprof.svg
|
||||
@$(GO) tool pprof --inuse_space -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/mem.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/memprof.inuse.svg
|
||||
@$(GO) tool pprof --alloc_space -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/mem.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/memprof.alloc.svg
|
||||
@$(GO) tool pprof -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/block.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/blockprof.svg
|
||||
@$(GO) tool pprof -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/mutex.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/mutexprof.svg
|
232
vendor/github.com/prometheus/tsdb/Makefile.common
generated
vendored
Normal file
232
vendor/github.com/prometheus/tsdb/Makefile.common
generated
vendored
Normal file
|
@ -0,0 +1,232 @@
|
|||
# Copyright 2018 The Prometheus Authors
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# A common Makefile that includes rules to be reused in different prometheus projects.
|
||||
# !!! Open PRs only against the prometheus/prometheus/Makefile.common repository!
|
||||
|
||||
# Example usage :
|
||||
# Create the main Makefile in the root project directory.
|
||||
# include Makefile.common
|
||||
# customTarget:
|
||||
# @echo ">> Running customTarget"
|
||||
#
|
||||
|
||||
# Ensure GOBIN is not set during build so that promu is installed to the correct path
|
||||
unexport GOBIN
|
||||
|
||||
GO ?= go
|
||||
GOFMT ?= $(GO)fmt
|
||||
FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH)))
|
||||
GOOPTS ?=
|
||||
GOHOSTOS ?= $(shell $(GO) env GOHOSTOS)
|
||||
GOHOSTARCH ?= $(shell $(GO) env GOHOSTARCH)
|
||||
|
||||
GO_VERSION ?= $(shell $(GO) version)
|
||||
GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION))
|
||||
PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.')
|
||||
|
||||
GOVENDOR :=
|
||||
GO111MODULE :=
|
||||
ifeq (, $(PRE_GO_111))
|
||||
ifneq (,$(wildcard go.mod))
|
||||
# Enforce Go modules support just in case the directory is inside GOPATH (and for Travis CI).
|
||||
GO111MODULE := on
|
||||
|
||||
ifneq (,$(wildcard vendor))
|
||||
# Always use the local vendor/ directory to satisfy the dependencies.
|
||||
GOOPTS := $(GOOPTS) -mod=vendor
|
||||
endif
|
||||
endif
|
||||
else
|
||||
ifneq (,$(wildcard go.mod))
|
||||
ifneq (,$(wildcard vendor))
|
||||
$(warning This repository requires Go >= 1.11 because of Go modules)
|
||||
$(warning Some recipes may not work as expected as the current Go runtime is '$(GO_VERSION_NUMBER)')
|
||||
endif
|
||||
else
|
||||
# This repository isn't using Go modules (yet).
|
||||
GOVENDOR := $(FIRST_GOPATH)/bin/govendor
|
||||
endif
|
||||
endif
|
||||
PROMU := $(FIRST_GOPATH)/bin/promu
|
||||
STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck
|
||||
pkgs = ./...
|
||||
|
||||
ifeq (arm, $(GOHOSTARCH))
|
||||
GOHOSTARM ?= $(shell GOARM= $(GO) env GOARM)
|
||||
GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)v$(GOHOSTARM)
|
||||
else
|
||||
GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)
|
||||
endif
|
||||
|
||||
PROMU_VERSION ?= 0.3.0
|
||||
PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
|
||||
STATICCHECK_VERSION ?= 2019.1
|
||||
STATICCHECK_URL := https://github.com/dominikh/go-tools/releases/download/$(STATICCHECK_VERSION)/staticcheck_$(GOHOSTOS)_$(GOHOSTARCH)
|
||||
|
||||
PREFIX ?= $(shell pwd)
|
||||
BIN_DIR ?= $(shell pwd)
|
||||
DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
|
||||
DOCKER_REPO ?= prom
|
||||
|
||||
ifeq ($(GOHOSTARCH),amd64)
|
||||
ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin windows))
|
||||
# Only supported on amd64
|
||||
test-flags := -race
|
||||
endif
|
||||
endif
|
||||
|
||||
# This rule is used to forward a target like "build" to "common-build". This
|
||||
# allows a new "build" target to be defined in a Makefile which includes this
|
||||
# one and override "common-build" without override warnings.
|
||||
%: common-% ;
|
||||
|
||||
.PHONY: common-all
|
||||
common-all: precheck style check_license staticcheck unused build test
|
||||
|
||||
.PHONY: common-style
|
||||
common-style:
|
||||
@echo ">> checking code style"
|
||||
@fmtRes=$$($(GOFMT) -d $$(find . -path ./vendor -prune -o -name '*.go' -print)); \
|
||||
if [ -n "$${fmtRes}" ]; then \
|
||||
echo "gofmt checking failed!"; echo "$${fmtRes}"; echo; \
|
||||
echo "Please ensure you are using $$($(GO) version) for formatting code."; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
.PHONY: common-check_license
|
||||
common-check_license:
|
||||
@echo ">> checking license header"
|
||||
@licRes=$$(for file in $$(find . -type f -iname '*.go' ! -path './vendor/*') ; do \
|
||||
awk 'NR<=3' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \
|
||||
done); \
|
||||
if [ -n "$${licRes}" ]; then \
|
||||
echo "license header checking failed:"; echo "$${licRes}"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
.PHONY: common-test-short
|
||||
common-test-short:
|
||||
@echo ">> running short tests"
|
||||
GO111MODULE=$(GO111MODULE) $(GO) test -short $(GOOPTS) $(pkgs)
|
||||
|
||||
.PHONY: common-test
|
||||
common-test:
|
||||
@echo ">> running all tests"
|
||||
GO111MODULE=$(GO111MODULE) $(GO) test $(test-flags) $(GOOPTS) $(pkgs)
|
||||
|
||||
.PHONY: common-format
|
||||
common-format:
|
||||
@echo ">> formatting code"
|
||||
GO111MODULE=$(GO111MODULE) $(GO) fmt $(pkgs)
|
||||
|
||||
.PHONY: common-vet
|
||||
common-vet:
|
||||
@echo ">> vetting code"
|
||||
GO111MODULE=$(GO111MODULE) $(GO) vet $(GOOPTS) $(pkgs)
|
||||
|
||||
.PHONY: common-staticcheck
|
||||
common-staticcheck: $(STATICCHECK)
|
||||
@echo ">> running staticcheck"
|
||||
chmod +x $(STATICCHECK)
|
||||
ifdef GO111MODULE
|
||||
# 'go list' needs to be executed before staticcheck to prepopulate the modules cache.
|
||||
# Otherwise staticcheck might fail randomly for some reason not yet explained.
|
||||
GO111MODULE=$(GO111MODULE) $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null
|
||||
GO111MODULE=$(GO111MODULE) $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs)
|
||||
else
|
||||
$(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs)
|
||||
endif
|
||||
|
||||
.PHONY: common-unused
|
||||
common-unused: $(GOVENDOR)
|
||||
ifdef GOVENDOR
|
||||
@echo ">> running check for unused packages"
|
||||
@$(GOVENDOR) list +unused | grep . && exit 1 || echo 'No unused packages'
|
||||
else
|
||||
ifdef GO111MODULE
|
||||
@echo ">> running check for unused/missing packages in go.mod"
|
||||
GO111MODULE=$(GO111MODULE) $(GO) mod tidy
|
||||
ifeq (,$(wildcard vendor))
|
||||
@git diff --exit-code -- go.sum go.mod
|
||||
else
|
||||
@echo ">> running check for unused packages in vendor/"
|
||||
GO111MODULE=$(GO111MODULE) $(GO) mod vendor
|
||||
@git diff --exit-code -- go.sum go.mod vendor/
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
|
||||
.PHONY: common-build
|
||||
common-build: promu
|
||||
@echo ">> building binaries"
|
||||
GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX)
|
||||
|
||||
.PHONY: common-tarball
|
||||
common-tarball: promu
|
||||
@echo ">> building release tarball"
|
||||
$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR)
|
||||
|
||||
.PHONY: common-docker
|
||||
common-docker:
|
||||
docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
|
||||
|
||||
.PHONY: common-docker-publish
|
||||
common-docker-publish:
|
||||
docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)"
|
||||
|
||||
.PHONY: common-docker-tag-latest
|
||||
common-docker-tag-latest:
|
||||
docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):latest"
|
||||
|
||||
.PHONY: promu
|
||||
promu: $(PROMU)
|
||||
|
||||
$(PROMU):
|
||||
$(eval PROMU_TMP := $(shell mktemp -d))
|
||||
curl -s -L $(PROMU_URL) | tar -xvzf - -C $(PROMU_TMP)
|
||||
mkdir -p $(FIRST_GOPATH)/bin
|
||||
cp $(PROMU_TMP)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(FIRST_GOPATH)/bin/promu
|
||||
rm -r $(PROMU_TMP)
|
||||
|
||||
.PHONY: proto
|
||||
proto:
|
||||
@echo ">> generating code from proto files"
|
||||
@./scripts/genproto.sh
|
||||
|
||||
$(STATICCHECK):
|
||||
mkdir -p $(FIRST_GOPATH)/bin
|
||||
curl -s -L $(STATICCHECK_URL) > $(STATICCHECK)
|
||||
|
||||
ifdef GOVENDOR
|
||||
.PHONY: $(GOVENDOR)
|
||||
$(GOVENDOR):
|
||||
GOOS= GOARCH= $(GO) get -u github.com/kardianos/govendor
|
||||
endif
|
||||
|
||||
.PHONY: precheck
|
||||
precheck::
|
||||
|
||||
define PRECHECK_COMMAND_template =
|
||||
precheck:: $(1)_precheck
|
||||
|
||||
|
||||
PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1)))
|
||||
.PHONY: $(1)_precheck
|
||||
$(1)_precheck:
|
||||
@if ! $$(PRECHECK_COMMAND_$(1)) 1>/dev/null 2>&1; then \
|
||||
echo "Execution of '$$(PRECHECK_COMMAND_$(1))' command failed. Is $(1) installed?"; \
|
||||
exit 1; \
|
||||
fi
|
||||
endef
|
14
vendor/github.com/prometheus/tsdb/README.md
generated
vendored
Normal file
14
vendor/github.com/prometheus/tsdb/README.md
generated
vendored
Normal file
|
@ -0,0 +1,14 @@
|
|||
# TSDB [](https://travis-ci.org/prometheus/tsdb)
|
||||
|
||||
[](https://godoc.org/github.com/prometheus/tsdb)
|
||||
[](https://goreportcard.com/report/github.com/prometheus/tsdb)
|
||||
|
||||
This repository contains the Prometheus storage layer that is used in its 2.x releases.
|
||||
|
||||
A writeup of its design can be found [here](https://fabxc.org/blog/2017-04-10-writing-a-tsdb/).
|
||||
|
||||
Based on the Gorilla TSDB [white papers](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf).
|
||||
|
||||
Video: [Storing 16 Bytes at Scale](https://youtu.be/b_pEevMAC3I) from [PromCon 2017](https://promcon.io/2017-munich/).
|
||||
|
||||
See also the [format documentation](docs/format/README.md).
|
640
vendor/github.com/prometheus/tsdb/block.go
generated
vendored
Normal file
640
vendor/github.com/prometheus/tsdb/block.go
generated
vendored
Normal file
|
@ -0,0 +1,640 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tsdb
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/go-kit/kit/log"
|
||||
"github.com/go-kit/kit/log/level"
|
||||
"github.com/oklog/ulid"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/tsdb/chunks"
|
||||
"github.com/prometheus/tsdb/index"
|
||||
"github.com/prometheus/tsdb/labels"
|
||||
)
|
||||
|
||||
// IndexWriter serializes the index for a block of series data.
|
||||
// The methods must be called in the order they are specified in.
|
||||
type IndexWriter interface {
|
||||
// AddSymbols registers all string symbols that are encountered in series
|
||||
// and other indices.
|
||||
AddSymbols(sym map[string]struct{}) error
|
||||
|
||||
// AddSeries populates the index writer with a series and its offsets
|
||||
// of chunks that the index can reference.
|
||||
// Implementations may require series to be insert in increasing order by
|
||||
// their labels.
|
||||
// The reference numbers are used to resolve entries in postings lists that
|
||||
// are added later.
|
||||
AddSeries(ref uint64, l labels.Labels, chunks ...chunks.Meta) error
|
||||
|
||||
// WriteLabelIndex serializes an index from label names to values.
|
||||
// The passed in values chained tuples of strings of the length of names.
|
||||
WriteLabelIndex(names []string, values []string) error
|
||||
|
||||
// WritePostings writes a postings list for a single label pair.
|
||||
// The Postings here contain refs to the series that were added.
|
||||
WritePostings(name, value string, it index.Postings) error
|
||||
|
||||
// Close writes any finalization and closes the resources associated with
|
||||
// the underlying writer.
|
||||
Close() error
|
||||
}
|
||||
|
||||
// IndexReader provides reading access of serialized index data.
|
||||
type IndexReader interface {
|
||||
// Symbols returns a set of string symbols that may occur in series' labels
|
||||
// and indices.
|
||||
Symbols() (map[string]struct{}, error)
|
||||
|
||||
// LabelValues returns the possible label values.
|
||||
LabelValues(names ...string) (index.StringTuples, error)
|
||||
|
||||
// Postings returns the postings list iterator for the label pair.
|
||||
// The Postings here contain the offsets to the series inside the index.
|
||||
// Found IDs are not strictly required to point to a valid Series, e.g. during
|
||||
// background garbage collections.
|
||||
Postings(name, value string) (index.Postings, error)
|
||||
|
||||
// SortedPostings returns a postings list that is reordered to be sorted
|
||||
// by the label set of the underlying series.
|
||||
SortedPostings(index.Postings) index.Postings
|
||||
|
||||
// Series populates the given labels and chunk metas for the series identified
|
||||
// by the reference.
|
||||
// Returns ErrNotFound if the ref does not resolve to a known series.
|
||||
Series(ref uint64, lset *labels.Labels, chks *[]chunks.Meta) error
|
||||
|
||||
// LabelIndices returns a list of string tuples for which a label value index exists.
|
||||
// NOTE: This is deprecated. Use `LabelNames()` instead.
|
||||
LabelIndices() ([][]string, error)
|
||||
|
||||
// LabelNames returns all the unique label names present in the index in sorted order.
|
||||
LabelNames() ([]string, error)
|
||||
|
||||
// Close releases the underlying resources of the reader.
|
||||
Close() error
|
||||
}
|
||||
|
||||
// StringTuples provides access to a sorted list of string tuples.
|
||||
type StringTuples interface {
|
||||
// Total number of tuples in the list.
|
||||
Len() int
|
||||
// At returns the tuple at position i.
|
||||
At(i int) ([]string, error)
|
||||
}
|
||||
|
||||
// ChunkWriter serializes a time block of chunked series data.
|
||||
type ChunkWriter interface {
|
||||
// WriteChunks writes several chunks. The Chunk field of the ChunkMetas
|
||||
// must be populated.
|
||||
// After returning successfully, the Ref fields in the ChunkMetas
|
||||
// are set and can be used to retrieve the chunks from the written data.
|
||||
WriteChunks(chunks ...chunks.Meta) error
|
||||
|
||||
// Close writes any required finalization and closes the resources
|
||||
// associated with the underlying writer.
|
||||
Close() error
|
||||
}
|
||||
|
||||
// ChunkReader provides reading access of serialized time series data.
|
||||
type ChunkReader interface {
|
||||
// Chunk returns the series data chunk with the given reference.
|
||||
Chunk(ref uint64) (chunkenc.Chunk, error)
|
||||
|
||||
// Close releases all underlying resources of the reader.
|
||||
Close() error
|
||||
}
|
||||
|
||||
// BlockReader provides reading access to a data block.
|
||||
type BlockReader interface {
|
||||
// Index returns an IndexReader over the block's data.
|
||||
Index() (IndexReader, error)
|
||||
|
||||
// Chunks returns a ChunkReader over the block's data.
|
||||
Chunks() (ChunkReader, error)
|
||||
|
||||
// Tombstones returns a TombstoneReader over the block's deleted data.
|
||||
Tombstones() (TombstoneReader, error)
|
||||
|
||||
// MinTime returns the min time of the block.
|
||||
MinTime() int64
|
||||
|
||||
// MaxTime returns the max time of the block.
|
||||
MaxTime() int64
|
||||
}
|
||||
|
||||
// Appendable defines an entity to which data can be appended.
|
||||
type Appendable interface {
|
||||
// Appender returns a new Appender against an underlying store.
|
||||
Appender() Appender
|
||||
}
|
||||
|
||||
// SizeReader returns the size of the object in bytes.
|
||||
type SizeReader interface {
|
||||
// Size returns the size in bytes.
|
||||
Size() int64
|
||||
}
|
||||
|
||||
// BlockMeta provides meta information about a block.
|
||||
type BlockMeta struct {
|
||||
// Unique identifier for the block and its contents. Changes on compaction.
|
||||
ULID ulid.ULID `json:"ulid"`
|
||||
|
||||
// MinTime and MaxTime specify the time range all samples
|
||||
// in the block are in.
|
||||
MinTime int64 `json:"minTime"`
|
||||
MaxTime int64 `json:"maxTime"`
|
||||
|
||||
// Stats about the contents of the block.
|
||||
Stats BlockStats `json:"stats,omitempty"`
|
||||
|
||||
// Information on compactions the block was created from.
|
||||
Compaction BlockMetaCompaction `json:"compaction"`
|
||||
|
||||
// Version of the index format.
|
||||
Version int `json:"version"`
|
||||
}
|
||||
|
||||
// BlockStats contains stats about contents of a block.
|
||||
type BlockStats struct {
|
||||
NumSamples uint64 `json:"numSamples,omitempty"`
|
||||
NumSeries uint64 `json:"numSeries,omitempty"`
|
||||
NumChunks uint64 `json:"numChunks,omitempty"`
|
||||
NumTombstones uint64 `json:"numTombstones,omitempty"`
|
||||
NumBytes int64 `json:"numBytes,omitempty"`
|
||||
}
|
||||
|
||||
// BlockDesc describes a block by ULID and time range.
|
||||
type BlockDesc struct {
|
||||
ULID ulid.ULID `json:"ulid"`
|
||||
MinTime int64 `json:"minTime"`
|
||||
MaxTime int64 `json:"maxTime"`
|
||||
}
|
||||
|
||||
// BlockMetaCompaction holds information about compactions a block went through.
|
||||
type BlockMetaCompaction struct {
|
||||
// Maximum number of compaction cycles any source block has
|
||||
// gone through.
|
||||
Level int `json:"level"`
|
||||
// ULIDs of all source head blocks that went into the block.
|
||||
Sources []ulid.ULID `json:"sources,omitempty"`
|
||||
// Indicates that during compaction it resulted in a block without any samples
|
||||
// so it should be deleted on the next reload.
|
||||
Deletable bool `json:"deletable,omitempty"`
|
||||
// Short descriptions of the direct blocks that were used to create
|
||||
// this block.
|
||||
Parents []BlockDesc `json:"parents,omitempty"`
|
||||
Failed bool `json:"failed,omitempty"`
|
||||
}
|
||||
|
||||
const indexFilename = "index"
|
||||
const metaFilename = "meta.json"
|
||||
|
||||
func chunkDir(dir string) string { return filepath.Join(dir, "chunks") }
|
||||
|
||||
func readMetaFile(dir string) (*BlockMeta, error) {
|
||||
b, err := ioutil.ReadFile(filepath.Join(dir, metaFilename))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m BlockMeta
|
||||
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m.Version != 1 {
|
||||
return nil, errors.Errorf("unexpected meta file version %d", m.Version)
|
||||
}
|
||||
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
func writeMetaFile(dir string, meta *BlockMeta) error {
|
||||
meta.Version = 1
|
||||
|
||||
// Make any changes to the file appear atomic.
|
||||
path := filepath.Join(dir, metaFilename)
|
||||
tmp := path + ".tmp"
|
||||
|
||||
f, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(f)
|
||||
enc.SetIndent("", "\t")
|
||||
|
||||
var merr MultiError
|
||||
|
||||
if merr.Add(enc.Encode(meta)); merr.Err() != nil {
|
||||
merr.Add(f.Close())
|
||||
return merr.Err()
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return renameFile(tmp, path)
|
||||
}
|
||||
|
||||
// Block represents a directory of time series data covering a continuous time range.
|
||||
type Block struct {
|
||||
mtx sync.RWMutex
|
||||
closing bool
|
||||
pendingReaders sync.WaitGroup
|
||||
|
||||
dir string
|
||||
meta BlockMeta
|
||||
|
||||
// Symbol Table Size in bytes.
|
||||
// We maintain this variable to avoid recalculation everytime.
|
||||
symbolTableSize uint64
|
||||
|
||||
chunkr ChunkReader
|
||||
indexr IndexReader
|
||||
tombstones TombstoneReader
|
||||
}
|
||||
|
||||
// OpenBlock opens the block in the directory. It can be passed a chunk pool, which is used
|
||||
// to instantiate chunk structs.
|
||||
func OpenBlock(logger log.Logger, dir string, pool chunkenc.Pool) (pb *Block, err error) {
|
||||
if logger == nil {
|
||||
logger = log.NewNopLogger()
|
||||
}
|
||||
var closers []io.Closer
|
||||
defer func() {
|
||||
if err != nil {
|
||||
var merr MultiError
|
||||
merr.Add(err)
|
||||
merr.Add(closeAll(closers))
|
||||
err = merr.Err()
|
||||
}
|
||||
}()
|
||||
meta, err := readMetaFile(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cr, err := chunks.NewDirReader(chunkDir(dir), pool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
closers = append(closers, cr)
|
||||
|
||||
ir, err := index.NewFileReader(filepath.Join(dir, indexFilename))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
closers = append(closers, ir)
|
||||
|
||||
tr, tsr, err := readTombstones(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
closers = append(closers, tr)
|
||||
|
||||
// TODO refactor to set this at block creation time as
|
||||
// that would be the logical place for a block size to be calculated.
|
||||
bs := blockSize(cr, ir, tsr)
|
||||
meta.Stats.NumBytes = bs
|
||||
err = writeMetaFile(dir, meta)
|
||||
if err != nil {
|
||||
level.Warn(logger).Log("msg", "couldn't write the meta file for the block size", "block", dir, "err", err)
|
||||
}
|
||||
|
||||
pb = &Block{
|
||||
dir: dir,
|
||||
meta: *meta,
|
||||
chunkr: cr,
|
||||
indexr: ir,
|
||||
tombstones: tr,
|
||||
symbolTableSize: ir.SymbolTableSize(),
|
||||
}
|
||||
return pb, nil
|
||||
}
|
||||
|
||||
func blockSize(rr ...SizeReader) int64 {
|
||||
var total int64
|
||||
for _, r := range rr {
|
||||
if r != nil {
|
||||
total += r.Size()
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// Close closes the on-disk block. It blocks as long as there are readers reading from the block.
|
||||
func (pb *Block) Close() error {
|
||||
pb.mtx.Lock()
|
||||
pb.closing = true
|
||||
pb.mtx.Unlock()
|
||||
|
||||
pb.pendingReaders.Wait()
|
||||
|
||||
var merr MultiError
|
||||
|
||||
merr.Add(pb.chunkr.Close())
|
||||
merr.Add(pb.indexr.Close())
|
||||
merr.Add(pb.tombstones.Close())
|
||||
|
||||
return merr.Err()
|
||||
}
|
||||
|
||||
func (pb *Block) String() string {
|
||||
return pb.meta.ULID.String()
|
||||
}
|
||||
|
||||
// Dir returns the directory of the block.
|
||||
func (pb *Block) Dir() string { return pb.dir }
|
||||
|
||||
// Meta returns meta information about the block.
|
||||
func (pb *Block) Meta() BlockMeta { return pb.meta }
|
||||
|
||||
// MinTime returns the min time of the meta.
|
||||
func (pb *Block) MinTime() int64 { return pb.meta.MinTime }
|
||||
|
||||
// MaxTime returns the max time of the meta.
|
||||
func (pb *Block) MaxTime() int64 { return pb.meta.MaxTime }
|
||||
|
||||
// Size returns the number of bytes that the block takes up.
|
||||
func (pb *Block) Size() int64 { return pb.meta.Stats.NumBytes }
|
||||
|
||||
// ErrClosing is returned when a block is in the process of being closed.
|
||||
var ErrClosing = errors.New("block is closing")
|
||||
|
||||
func (pb *Block) startRead() error {
|
||||
pb.mtx.RLock()
|
||||
defer pb.mtx.RUnlock()
|
||||
|
||||
if pb.closing {
|
||||
return ErrClosing
|
||||
}
|
||||
pb.pendingReaders.Add(1)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Index returns a new IndexReader against the block data.
|
||||
func (pb *Block) Index() (IndexReader, error) {
|
||||
if err := pb.startRead(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blockIndexReader{ir: pb.indexr, b: pb}, nil
|
||||
}
|
||||
|
||||
// Chunks returns a new ChunkReader against the block data.
|
||||
func (pb *Block) Chunks() (ChunkReader, error) {
|
||||
if err := pb.startRead(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blockChunkReader{ChunkReader: pb.chunkr, b: pb}, nil
|
||||
}
|
||||
|
||||
// Tombstones returns a new TombstoneReader against the block data.
|
||||
func (pb *Block) Tombstones() (TombstoneReader, error) {
|
||||
if err := pb.startRead(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blockTombstoneReader{TombstoneReader: pb.tombstones, b: pb}, nil
|
||||
}
|
||||
|
||||
// GetSymbolTableSize returns the Symbol Table Size in the index of this block.
|
||||
func (pb *Block) GetSymbolTableSize() uint64 {
|
||||
return pb.symbolTableSize
|
||||
}
|
||||
|
||||
func (pb *Block) setCompactionFailed() error {
|
||||
pb.meta.Compaction.Failed = true
|
||||
return writeMetaFile(pb.dir, &pb.meta)
|
||||
}
|
||||
|
||||
type blockIndexReader struct {
|
||||
ir IndexReader
|
||||
b *Block
|
||||
}
|
||||
|
||||
func (r blockIndexReader) Symbols() (map[string]struct{}, error) {
|
||||
s, err := r.ir.Symbols()
|
||||
return s, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
|
||||
}
|
||||
|
||||
func (r blockIndexReader) LabelValues(names ...string) (index.StringTuples, error) {
|
||||
st, err := r.ir.LabelValues(names...)
|
||||
return st, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
|
||||
}
|
||||
|
||||
func (r blockIndexReader) Postings(name, value string) (index.Postings, error) {
|
||||
p, err := r.ir.Postings(name, value)
|
||||
return p, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
|
||||
}
|
||||
|
||||
func (r blockIndexReader) SortedPostings(p index.Postings) index.Postings {
|
||||
return r.ir.SortedPostings(p)
|
||||
}
|
||||
|
||||
func (r blockIndexReader) Series(ref uint64, lset *labels.Labels, chks *[]chunks.Meta) error {
|
||||
return errors.Wrapf(
|
||||
r.ir.Series(ref, lset, chks),
|
||||
"block: %s",
|
||||
r.b.Meta().ULID,
|
||||
)
|
||||
}
|
||||
|
||||
func (r blockIndexReader) LabelIndices() ([][]string, error) {
|
||||
ss, err := r.ir.LabelIndices()
|
||||
return ss, errors.Wrapf(err, "block: %s", r.b.Meta().ULID)
|
||||
}
|
||||
|
||||
func (r blockIndexReader) LabelNames() ([]string, error) {
|
||||
return r.b.LabelNames()
|
||||
}
|
||||
|
||||
func (r blockIndexReader) Close() error {
|
||||
r.b.pendingReaders.Done()
|
||||
return nil
|
||||
}
|
||||
|
||||
type blockTombstoneReader struct {
|
||||
TombstoneReader
|
||||
b *Block
|
||||
}
|
||||
|
||||
func (r blockTombstoneReader) Close() error {
|
||||
r.b.pendingReaders.Done()
|
||||
return nil
|
||||
}
|
||||
|
||||
type blockChunkReader struct {
|
||||
ChunkReader
|
||||
b *Block
|
||||
}
|
||||
|
||||
func (r blockChunkReader) Close() error {
|
||||
r.b.pendingReaders.Done()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete matching series between mint and maxt in the block.
|
||||
func (pb *Block) Delete(mint, maxt int64, ms ...labels.Matcher) error {
|
||||
pb.mtx.Lock()
|
||||
defer pb.mtx.Unlock()
|
||||
|
||||
if pb.closing {
|
||||
return ErrClosing
|
||||
}
|
||||
|
||||
p, err := PostingsForMatchers(pb.indexr, ms...)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "select series")
|
||||
}
|
||||
|
||||
ir := pb.indexr
|
||||
|
||||
// Choose only valid postings which have chunks in the time-range.
|
||||
stones := newMemTombstones()
|
||||
|
||||
var lset labels.Labels
|
||||
var chks []chunks.Meta
|
||||
|
||||
Outer:
|
||||
for p.Next() {
|
||||
err := ir.Series(p.At(), &lset, &chks)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, chk := range chks {
|
||||
if chk.OverlapsClosedInterval(mint, maxt) {
|
||||
// Delete only until the current values and not beyond.
|
||||
tmin, tmax := clampInterval(mint, maxt, chks[0].MinTime, chks[len(chks)-1].MaxTime)
|
||||
stones.addInterval(p.At(), Interval{tmin, tmax})
|
||||
continue Outer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if p.Err() != nil {
|
||||
return p.Err()
|
||||
}
|
||||
|
||||
err = pb.tombstones.Iter(func(id uint64, ivs Intervals) error {
|
||||
for _, iv := range ivs {
|
||||
stones.addInterval(id, iv)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pb.tombstones = stones
|
||||
pb.meta.Stats.NumTombstones = pb.tombstones.Total()
|
||||
|
||||
if err := writeTombstoneFile(pb.dir, pb.tombstones); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeMetaFile(pb.dir, &pb.meta)
|
||||
}
|
||||
|
||||
// CleanTombstones will remove the tombstones and rewrite the block (only if there are any tombstones).
|
||||
// If there was a rewrite, then it returns the ULID of the new block written, else nil.
|
||||
func (pb *Block) CleanTombstones(dest string, c Compactor) (*ulid.ULID, error) {
|
||||
numStones := 0
|
||||
|
||||
if err := pb.tombstones.Iter(func(id uint64, ivs Intervals) error {
|
||||
numStones += len(ivs)
|
||||
return nil
|
||||
}); err != nil {
|
||||
// This should never happen, as the iteration function only returns nil.
|
||||
panic(err)
|
||||
}
|
||||
if numStones == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
meta := pb.Meta()
|
||||
uid, err := c.Write(dest, pb, pb.meta.MinTime, pb.meta.MaxTime, &meta)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &uid, nil
|
||||
}
|
||||
|
||||
// Snapshot creates snapshot of the block into dir.
|
||||
func (pb *Block) Snapshot(dir string) error {
|
||||
blockDir := filepath.Join(dir, pb.meta.ULID.String())
|
||||
if err := os.MkdirAll(blockDir, 0777); err != nil {
|
||||
return errors.Wrap(err, "create snapshot block dir")
|
||||
}
|
||||
|
||||
chunksDir := chunkDir(blockDir)
|
||||
if err := os.MkdirAll(chunksDir, 0777); err != nil {
|
||||
return errors.Wrap(err, "create snapshot chunk dir")
|
||||
}
|
||||
|
||||
// Hardlink meta, index and tombstones
|
||||
for _, fname := range []string{
|
||||
metaFilename,
|
||||
indexFilename,
|
||||
tombstoneFilename,
|
||||
} {
|
||||
if err := os.Link(filepath.Join(pb.dir, fname), filepath.Join(blockDir, fname)); err != nil {
|
||||
return errors.Wrapf(err, "create snapshot %s", fname)
|
||||
}
|
||||
}
|
||||
|
||||
// Hardlink the chunks
|
||||
curChunkDir := chunkDir(pb.dir)
|
||||
files, err := ioutil.ReadDir(curChunkDir)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "ReadDir the current chunk dir")
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
err := os.Link(filepath.Join(curChunkDir, f.Name()), filepath.Join(chunksDir, f.Name()))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "hardlink a chunk")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// OverlapsClosedInterval returns true if the block overlaps [mint, maxt].
|
||||
func (pb *Block) OverlapsClosedInterval(mint, maxt int64) bool {
|
||||
// The block itself is a half-open interval
|
||||
// [pb.meta.MinTime, pb.meta.MaxTime).
|
||||
return pb.meta.MinTime <= maxt && mint < pb.meta.MaxTime
|
||||
}
|
||||
|
||||
// LabelNames returns all the unique label names present in the Block in sorted order.
|
||||
func (pb *Block) LabelNames() ([]string, error) {
|
||||
return pb.indexr.LabelNames()
|
||||
}
|
||||
|
||||
func clampInterval(a, b, mint, maxt int64) (int64, int64) {
|
||||
if a < mint {
|
||||
a = mint
|
||||
}
|
||||
if b > maxt {
|
||||
b = maxt
|
||||
}
|
||||
return a, b
|
||||
}
|
260
vendor/github.com/prometheus/tsdb/checkpoint.go
generated
vendored
Normal file
260
vendor/github.com/prometheus/tsdb/checkpoint.go
generated
vendored
Normal file
|
@ -0,0 +1,260 @@
|
|||
// Copyright 2018 The Prometheus Authors
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tsdb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/tsdb/fileutil"
|
||||
"github.com/prometheus/tsdb/wal"
|
||||
)
|
||||
|
||||
// CheckpointStats returns stats about a created checkpoint.
|
||||
type CheckpointStats struct {
|
||||
DroppedSeries int
|
||||
DroppedSamples int
|
||||
DroppedTombstones int
|
||||
TotalSeries int // Processed series including dropped ones.
|
||||
TotalSamples int // Processed samples including dropped ones.
|
||||
TotalTombstones int // Processed tombstones including dropped ones.
|
||||
}
|
||||
|
||||
// LastCheckpoint returns the directory name and index of the most recent checkpoint.
|
||||
// If dir does not contain any checkpoints, ErrNotFound is returned.
|
||||
func LastCheckpoint(dir string) (string, int, error) {
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
// Traverse list backwards since there may be multiple checkpoints left.
|
||||
for i := len(files) - 1; i >= 0; i-- {
|
||||
fi := files[i]
|
||||
|
||||
if !strings.HasPrefix(fi.Name(), checkpointPrefix) {
|
||||
continue
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return "", 0, errors.Errorf("checkpoint %s is not a directory", fi.Name())
|
||||
}
|
||||
idx, err := strconv.Atoi(fi.Name()[len(checkpointPrefix):])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
return filepath.Join(dir, fi.Name()), idx, nil
|
||||
}
|
||||
return "", 0, ErrNotFound
|
||||
}
|
||||
|
||||
// DeleteCheckpoints deletes all checkpoints in a directory below a given index.
|
||||
func DeleteCheckpoints(dir string, maxIndex int) error {
|
||||
var errs MultiError
|
||||
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, fi := range files {
|
||||
if !strings.HasPrefix(fi.Name(), checkpointPrefix) {
|
||||
continue
|
||||
}
|
||||
index, err := strconv.Atoi(fi.Name()[len(checkpointPrefix):])
|
||||
if err != nil || index >= maxIndex {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(dir, fi.Name())); err != nil {
|
||||
errs.Add(err)
|
||||
}
|
||||
}
|
||||
return errs.Err()
|
||||
}
|
||||
|
||||
const checkpointPrefix = "checkpoint."
|
||||
|
||||
// Checkpoint creates a compacted checkpoint of segments in range [first, last] in the given WAL.
|
||||
// It includes the most recent checkpoint if it exists.
|
||||
// All series not satisfying keep and samples below mint are dropped.
|
||||
//
|
||||
// The checkpoint is stored in a directory named checkpoint.N in the same
|
||||
// segmented format as the original WAL itself.
|
||||
// This makes it easy to read it through the WAL package and concatenate
|
||||
// it with the original WAL.
|
||||
func Checkpoint(w *wal.WAL, from, to int, keep func(id uint64) bool, mint int64) (*CheckpointStats, error) {
|
||||
stats := &CheckpointStats{}
|
||||
var sgmReader io.ReadCloser
|
||||
|
||||
{
|
||||
|
||||
var sgmRange []wal.SegmentRange
|
||||
dir, idx, err := LastCheckpoint(w.Dir())
|
||||
if err != nil && err != ErrNotFound {
|
||||
return nil, errors.Wrap(err, "find last checkpoint")
|
||||
}
|
||||
last := idx + 1
|
||||
if err == nil {
|
||||
if from > last {
|
||||
return nil, fmt.Errorf("unexpected gap to last checkpoint. expected:%v, requested:%v", last, from)
|
||||
}
|
||||
// Ignore WAL files below the checkpoint. They shouldn't exist to begin with.
|
||||
from = last
|
||||
|
||||
sgmRange = append(sgmRange, wal.SegmentRange{Dir: dir, Last: math.MaxInt32})
|
||||
}
|
||||
|
||||
sgmRange = append(sgmRange, wal.SegmentRange{Dir: w.Dir(), First: from, Last: to})
|
||||
sgmReader, err = wal.NewSegmentsRangeReader(sgmRange...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "create segment reader")
|
||||
}
|
||||
defer sgmReader.Close()
|
||||
}
|
||||
|
||||
cpdir := filepath.Join(w.Dir(), fmt.Sprintf(checkpointPrefix+"%06d", to))
|
||||
cpdirtmp := cpdir + ".tmp"
|
||||
|
||||
if err := os.MkdirAll(cpdirtmp, 0777); err != nil {
|
||||
return nil, errors.Wrap(err, "create checkpoint dir")
|
||||
}
|
||||
cp, err := wal.New(nil, nil, cpdirtmp)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "open checkpoint")
|
||||
}
|
||||
|
||||
// Ensures that an early return caused by an error doesn't leave any tmp files.
|
||||
defer func() {
|
||||
cp.Close()
|
||||
os.RemoveAll(cpdirtmp)
|
||||
}()
|
||||
|
||||
r := wal.NewReader(sgmReader)
|
||||
|
||||
var (
|
||||
series []RefSeries
|
||||
samples []RefSample
|
||||
tstones []Stone
|
||||
dec RecordDecoder
|
||||
enc RecordEncoder
|
||||
buf []byte
|
||||
recs [][]byte
|
||||
)
|
||||
for r.Next() {
|
||||
series, samples, tstones = series[:0], samples[:0], tstones[:0]
|
||||
|
||||
// We don't reset the buffer since we batch up multiple records
|
||||
// before writing them to the checkpoint.
|
||||
// Remember where the record for this iteration starts.
|
||||
start := len(buf)
|
||||
rec := r.Record()
|
||||
|
||||
switch dec.Type(rec) {
|
||||
case RecordSeries:
|
||||
series, err = dec.Series(rec, series)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "decode series")
|
||||
}
|
||||
// Drop irrelevant series in place.
|
||||
repl := series[:0]
|
||||
for _, s := range series {
|
||||
if keep(s.Ref) {
|
||||
repl = append(repl, s)
|
||||
}
|
||||
}
|
||||
if len(repl) > 0 {
|
||||
buf = enc.Series(repl, buf)
|
||||
}
|
||||
stats.TotalSeries += len(series)
|
||||
stats.DroppedSeries += len(series) - len(repl)
|
||||
|
||||
case RecordSamples:
|
||||
samples, err = dec.Samples(rec, samples)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "decode samples")
|
||||
}
|
||||
// Drop irrelevant samples in place.
|
||||
repl := samples[:0]
|
||||
for _, s := range samples {
|
||||
if s.T >= mint {
|
||||
repl = append(repl, s)
|
||||
}
|
||||
}
|
||||
if len(repl) > 0 {
|
||||
buf = enc.Samples(repl, buf)
|
||||
}
|
||||
stats.TotalSamples += len(samples)
|
||||
stats.DroppedSamples += len(samples) - len(repl)
|
||||
|
||||
case RecordTombstones:
|
||||
tstones, err = dec.Tombstones(rec, tstones)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "decode deletes")
|
||||
}
|
||||
// Drop irrelevant tombstones in place.
|
||||
repl := tstones[:0]
|
||||
for _, s := range tstones {
|
||||
for _, iv := range s.intervals {
|
||||
if iv.Maxt >= mint {
|
||||
repl = append(repl, s)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(repl) > 0 {
|
||||
buf = enc.Tombstones(repl, buf)
|
||||
}
|
||||
stats.TotalTombstones += len(tstones)
|
||||
stats.DroppedTombstones += len(tstones) - len(repl)
|
||||
|
||||
default:
|
||||
return nil, errors.New("invalid record type")
|
||||
}
|
||||
if len(buf[start:]) == 0 {
|
||||
continue // All contents discarded.
|
||||
}
|
||||
recs = append(recs, buf[start:])
|
||||
|
||||
// Flush records in 1 MB increments.
|
||||
if len(buf) > 1*1024*1024 {
|
||||
if err := cp.Log(recs...); err != nil {
|
||||
return nil, errors.Wrap(err, "flush records")
|
||||
}
|
||||
buf, recs = buf[:0], recs[:0]
|
||||
}
|
||||
}
|
||||
// If we hit any corruption during checkpointing, repairing is not an option.
|
||||
// The head won't know which series records are lost.
|
||||
if r.Err() != nil {
|
||||
return nil, errors.Wrap(r.Err(), "read segments")
|
||||
}
|
||||
|
||||
// Flush remaining records.
|
||||
if err := cp.Log(recs...); err != nil {
|
||||
return nil, errors.Wrap(err, "flush records")
|
||||
}
|
||||
if err := cp.Close(); err != nil {
|
||||
return nil, errors.Wrap(err, "close checkpoint")
|
||||
}
|
||||
if err := fileutil.Replace(cpdirtmp, cpdir); err != nil {
|
||||
return nil, errors.Wrap(err, "rename checkpoint directory")
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
200
vendor/github.com/prometheus/tsdb/chunkenc/bstream.go
generated
vendored
Normal file
200
vendor/github.com/prometheus/tsdb/chunkenc/bstream.go
generated
vendored
Normal file
|
@ -0,0 +1,200 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The code in this file was largely written by Damian Gryski as part of
|
||||
// https://github.com/dgryski/go-tsz and published under the license below.
|
||||
// It received minor modifications to suit Prometheus's needs.
|
||||
|
||||
// Copyright (c) 2015,2016 Damian Gryski <damian@gryski.com>
|
||||
// All rights reserved.
|
||||
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
|
||||
// * Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// * Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import "io"
|
||||
|
||||
// bstream is a stream of bits.
|
||||
type bstream struct {
|
||||
stream []byte // the data stream
|
||||
count uint8 // how many bits are valid in current byte
|
||||
}
|
||||
|
||||
func newBReader(b []byte) bstream {
|
||||
return bstream{stream: b, count: 8}
|
||||
}
|
||||
|
||||
func (b *bstream) bytes() []byte {
|
||||
return b.stream
|
||||
}
|
||||
|
||||
type bit bool
|
||||
|
||||
const (
|
||||
zero bit = false
|
||||
one bit = true
|
||||
)
|
||||
|
||||
func (b *bstream) writeBit(bit bit) {
|
||||
if b.count == 0 {
|
||||
b.stream = append(b.stream, 0)
|
||||
b.count = 8
|
||||
}
|
||||
|
||||
i := len(b.stream) - 1
|
||||
|
||||
if bit {
|
||||
b.stream[i] |= 1 << (b.count - 1)
|
||||
}
|
||||
|
||||
b.count--
|
||||
}
|
||||
|
||||
func (b *bstream) writeByte(byt byte) {
|
||||
if b.count == 0 {
|
||||
b.stream = append(b.stream, 0)
|
||||
b.count = 8
|
||||
}
|
||||
|
||||
i := len(b.stream) - 1
|
||||
|
||||
// fill up b.b with b.count bits from byt
|
||||
b.stream[i] |= byt >> (8 - b.count)
|
||||
|
||||
b.stream = append(b.stream, 0)
|
||||
i++
|
||||
b.stream[i] = byt << b.count
|
||||
}
|
||||
|
||||
func (b *bstream) writeBits(u uint64, nbits int) {
|
||||
u <<= (64 - uint(nbits))
|
||||
for nbits >= 8 {
|
||||
byt := byte(u >> 56)
|
||||
b.writeByte(byt)
|
||||
u <<= 8
|
||||
nbits -= 8
|
||||
}
|
||||
|
||||
for nbits > 0 {
|
||||
b.writeBit((u >> 63) == 1)
|
||||
u <<= 1
|
||||
nbits--
|
||||
}
|
||||
}
|
||||
|
||||
func (b *bstream) readBit() (bit, error) {
|
||||
if len(b.stream) == 0 {
|
||||
return false, io.EOF
|
||||
}
|
||||
|
||||
if b.count == 0 {
|
||||
b.stream = b.stream[1:]
|
||||
|
||||
if len(b.stream) == 0 {
|
||||
return false, io.EOF
|
||||
}
|
||||
b.count = 8
|
||||
}
|
||||
|
||||
d := (b.stream[0] << (8 - b.count)) & 0x80
|
||||
b.count--
|
||||
return d != 0, nil
|
||||
}
|
||||
|
||||
func (b *bstream) ReadByte() (byte, error) {
|
||||
return b.readByte()
|
||||
}
|
||||
|
||||
func (b *bstream) readByte() (byte, error) {
|
||||
if len(b.stream) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
if b.count == 0 {
|
||||
b.stream = b.stream[1:]
|
||||
|
||||
if len(b.stream) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
return b.stream[0], nil
|
||||
}
|
||||
|
||||
if b.count == 8 {
|
||||
b.count = 0
|
||||
return b.stream[0], nil
|
||||
}
|
||||
|
||||
byt := b.stream[0] << (8 - b.count)
|
||||
b.stream = b.stream[1:]
|
||||
|
||||
if len(b.stream) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
// We just advanced the stream and can assume the shift to be 0.
|
||||
byt |= b.stream[0] >> b.count
|
||||
|
||||
return byt, nil
|
||||
}
|
||||
|
||||
func (b *bstream) readBits(nbits int) (uint64, error) {
|
||||
var u uint64
|
||||
|
||||
for nbits >= 8 {
|
||||
byt, err := b.readByte()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
u = (u << 8) | uint64(byt)
|
||||
nbits -= 8
|
||||
}
|
||||
|
||||
if nbits == 0 {
|
||||
return u, nil
|
||||
}
|
||||
|
||||
if nbits > int(b.count) {
|
||||
u = (u << uint(b.count)) | uint64((b.stream[0]<<(8-b.count))>>(8-b.count))
|
||||
nbits -= int(b.count)
|
||||
b.stream = b.stream[1:]
|
||||
|
||||
if len(b.stream) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
b.count = 8
|
||||
}
|
||||
|
||||
u = (u << uint(nbits)) | uint64((b.stream[0]<<(8-b.count))>>(8-uint(nbits)))
|
||||
b.count -= uint8(nbits)
|
||||
return u, nil
|
||||
}
|
131
vendor/github.com/prometheus/tsdb/chunkenc/chunk.go
generated
vendored
Normal file
131
vendor/github.com/prometheus/tsdb/chunkenc/chunk.go
generated
vendored
Normal file
|
@ -0,0 +1,131 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Encoding is the identifier for a chunk encoding.
|
||||
type Encoding uint8
|
||||
|
||||
func (e Encoding) String() string {
|
||||
switch e {
|
||||
case EncNone:
|
||||
return "none"
|
||||
case EncXOR:
|
||||
return "XOR"
|
||||
}
|
||||
return "<unknown>"
|
||||
}
|
||||
|
||||
// The different available chunk encodings.
|
||||
const (
|
||||
EncNone Encoding = iota
|
||||
EncXOR
|
||||
)
|
||||
|
||||
// Chunk holds a sequence of sample pairs that can be iterated over and appended to.
|
||||
type Chunk interface {
|
||||
Bytes() []byte
|
||||
Encoding() Encoding
|
||||
Appender() (Appender, error)
|
||||
Iterator() Iterator
|
||||
NumSamples() int
|
||||
}
|
||||
|
||||
// FromData returns a chunk from a byte slice of chunk data.
|
||||
func FromData(e Encoding, d []byte) (Chunk, error) {
|
||||
switch e {
|
||||
case EncXOR:
|
||||
return &XORChunk{b: bstream{count: 0, stream: d}}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unknown chunk encoding: %d", e)
|
||||
}
|
||||
|
||||
// Appender adds sample pairs to a chunk.
|
||||
type Appender interface {
|
||||
Append(int64, float64)
|
||||
}
|
||||
|
||||
// Iterator is a simple iterator that can only get the next value.
|
||||
type Iterator interface {
|
||||
At() (int64, float64)
|
||||
Err() error
|
||||
Next() bool
|
||||
}
|
||||
|
||||
// NewNopIterator returns a new chunk iterator that does not hold any data.
|
||||
func NewNopIterator() Iterator {
|
||||
return nopIterator{}
|
||||
}
|
||||
|
||||
type nopIterator struct{}
|
||||
|
||||
func (nopIterator) At() (int64, float64) { return 0, 0 }
|
||||
func (nopIterator) Next() bool { return false }
|
||||
func (nopIterator) Err() error { return nil }
|
||||
|
||||
type Pool interface {
|
||||
Put(Chunk) error
|
||||
Get(e Encoding, b []byte) (Chunk, error)
|
||||
}
|
||||
|
||||
// Pool is a memory pool of chunk objects.
|
||||
type pool struct {
|
||||
xor sync.Pool
|
||||
}
|
||||
|
||||
func NewPool() Pool {
|
||||
return &pool{
|
||||
xor: sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &XORChunk{b: bstream{}}
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *pool) Get(e Encoding, b []byte) (Chunk, error) {
|
||||
switch e {
|
||||
case EncXOR:
|
||||
c := p.xor.Get().(*XORChunk)
|
||||
c.b.stream = b
|
||||
c.b.count = 0
|
||||
return c, nil
|
||||
}
|
||||
return nil, errors.Errorf("invalid encoding %q", e)
|
||||
}
|
||||
|
||||
func (p *pool) Put(c Chunk) error {
|
||||
switch c.Encoding() {
|
||||
case EncXOR:
|
||||
xc, ok := c.(*XORChunk)
|
||||
// This may happen often with wrapped chunks. Nothing we can really do about
|
||||
// it but returning an error would cause a lot of allocations again. Thus,
|
||||
// we just skip it.
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
xc.b.stream = nil
|
||||
xc.b.count = 0
|
||||
p.xor.Put(c)
|
||||
default:
|
||||
return errors.Errorf("invalid encoding %q", c.Encoding())
|
||||
}
|
||||
return nil
|
||||
}
|
386
vendor/github.com/prometheus/tsdb/chunkenc/xor.go
generated
vendored
Normal file
386
vendor/github.com/prometheus/tsdb/chunkenc/xor.go
generated
vendored
Normal file
|
@ -0,0 +1,386 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The code in this file was largely written by Damian Gryski as part of
|
||||
// https://github.com/dgryski/go-tsz and published under the license below.
|
||||
// It was modified to accommodate reading from byte slices without modifying
|
||||
// the underlying bytes, which would panic when reading from mmaped
|
||||
// read-only byte slices.
|
||||
|
||||
// Copyright (c) 2015,2016 Damian Gryski <damian@gryski.com>
|
||||
// All rights reserved.
|
||||
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
|
||||
// * Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// * Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"math/bits"
|
||||
)
|
||||
|
||||
// XORChunk holds XOR encoded sample data.
|
||||
type XORChunk struct {
|
||||
b bstream
|
||||
}
|
||||
|
||||
// NewXORChunk returns a new chunk with XOR encoding of the given size.
|
||||
func NewXORChunk() *XORChunk {
|
||||
b := make([]byte, 2, 128)
|
||||
return &XORChunk{b: bstream{stream: b, count: 0}}
|
||||
}
|
||||
|
||||
// Encoding returns the encoding type.
|
||||
func (c *XORChunk) Encoding() Encoding {
|
||||
return EncXOR
|
||||
}
|
||||
|
||||
// Bytes returns the underlying byte slice of the chunk.
|
||||
func (c *XORChunk) Bytes() []byte {
|
||||
return c.b.bytes()
|
||||
}
|
||||
|
||||
// NumSamples returns the number of samples in the chunk.
|
||||
func (c *XORChunk) NumSamples() int {
|
||||
return int(binary.BigEndian.Uint16(c.Bytes()))
|
||||
}
|
||||
|
||||
// Appender implements the Chunk interface.
|
||||
func (c *XORChunk) Appender() (Appender, error) {
|
||||
it := c.iterator()
|
||||
|
||||
// To get an appender we must know the state it would have if we had
|
||||
// appended all existing data from scratch.
|
||||
// We iterate through the end and populate via the iterator's state.
|
||||
for it.Next() {
|
||||
}
|
||||
if err := it.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
a := &xorAppender{
|
||||
b: &c.b,
|
||||
t: it.t,
|
||||
v: it.val,
|
||||
tDelta: it.tDelta,
|
||||
leading: it.leading,
|
||||
trailing: it.trailing,
|
||||
}
|
||||
if binary.BigEndian.Uint16(a.b.bytes()) == 0 {
|
||||
a.leading = 0xff
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (c *XORChunk) iterator() *xorIterator {
|
||||
// Should iterators guarantee to act on a copy of the data so it doesn't lock append?
|
||||
// When using striped locks to guard access to chunks, probably yes.
|
||||
// Could only copy data if the chunk is not completed yet.
|
||||
return &xorIterator{
|
||||
br: newBReader(c.b.bytes()[2:]),
|
||||
numTotal: binary.BigEndian.Uint16(c.b.bytes()),
|
||||
}
|
||||
}
|
||||
|
||||
// Iterator implements the Chunk interface.
|
||||
func (c *XORChunk) Iterator() Iterator {
|
||||
return c.iterator()
|
||||
}
|
||||
|
||||
type xorAppender struct {
|
||||
b *bstream
|
||||
|
||||
t int64
|
||||
v float64
|
||||
tDelta uint64
|
||||
|
||||
leading uint8
|
||||
trailing uint8
|
||||
}
|
||||
|
||||
func (a *xorAppender) Append(t int64, v float64) {
|
||||
var tDelta uint64
|
||||
num := binary.BigEndian.Uint16(a.b.bytes())
|
||||
|
||||
if num == 0 {
|
||||
buf := make([]byte, binary.MaxVarintLen64)
|
||||
for _, b := range buf[:binary.PutVarint(buf, t)] {
|
||||
a.b.writeByte(b)
|
||||
}
|
||||
a.b.writeBits(math.Float64bits(v), 64)
|
||||
|
||||
} else if num == 1 {
|
||||
tDelta = uint64(t - a.t)
|
||||
|
||||
buf := make([]byte, binary.MaxVarintLen64)
|
||||
for _, b := range buf[:binary.PutUvarint(buf, tDelta)] {
|
||||
a.b.writeByte(b)
|
||||
}
|
||||
|
||||
a.writeVDelta(v)
|
||||
|
||||
} else {
|
||||
tDelta = uint64(t - a.t)
|
||||
dod := int64(tDelta - a.tDelta)
|
||||
|
||||
// Gorilla has a max resolution of seconds, Prometheus milliseconds.
|
||||
// Thus we use higher value range steps with larger bit size.
|
||||
switch {
|
||||
case dod == 0:
|
||||
a.b.writeBit(zero)
|
||||
case bitRange(dod, 14):
|
||||
a.b.writeBits(0x02, 2) // '10'
|
||||
a.b.writeBits(uint64(dod), 14)
|
||||
case bitRange(dod, 17):
|
||||
a.b.writeBits(0x06, 3) // '110'
|
||||
a.b.writeBits(uint64(dod), 17)
|
||||
case bitRange(dod, 20):
|
||||
a.b.writeBits(0x0e, 4) // '1110'
|
||||
a.b.writeBits(uint64(dod), 20)
|
||||
default:
|
||||
a.b.writeBits(0x0f, 4) // '1111'
|
||||
a.b.writeBits(uint64(dod), 64)
|
||||
}
|
||||
|
||||
a.writeVDelta(v)
|
||||
}
|
||||
|
||||
a.t = t
|
||||
a.v = v
|
||||
binary.BigEndian.PutUint16(a.b.bytes(), num+1)
|
||||
a.tDelta = tDelta
|
||||
}
|
||||
|
||||
func bitRange(x int64, nbits uint8) bool {
|
||||
return -((1<<(nbits-1))-1) <= x && x <= 1<<(nbits-1)
|
||||
}
|
||||
|
||||
func (a *xorAppender) writeVDelta(v float64) {
|
||||
vDelta := math.Float64bits(v) ^ math.Float64bits(a.v)
|
||||
|
||||
if vDelta == 0 {
|
||||
a.b.writeBit(zero)
|
||||
return
|
||||
}
|
||||
a.b.writeBit(one)
|
||||
|
||||
leading := uint8(bits.LeadingZeros64(vDelta))
|
||||
trailing := uint8(bits.TrailingZeros64(vDelta))
|
||||
|
||||
// Clamp number of leading zeros to avoid overflow when encoding.
|
||||
if leading >= 32 {
|
||||
leading = 31
|
||||
}
|
||||
|
||||
if a.leading != 0xff && leading >= a.leading && trailing >= a.trailing {
|
||||
a.b.writeBit(zero)
|
||||
a.b.writeBits(vDelta>>a.trailing, 64-int(a.leading)-int(a.trailing))
|
||||
} else {
|
||||
a.leading, a.trailing = leading, trailing
|
||||
|
||||
a.b.writeBit(one)
|
||||
a.b.writeBits(uint64(leading), 5)
|
||||
|
||||
// Note that if leading == trailing == 0, then sigbits == 64. But that value doesn't actually fit into the 6 bits we have.
|
||||
// Luckily, we never need to encode 0 significant bits, since that would put us in the other case (vdelta == 0).
|
||||
// So instead we write out a 0 and adjust it back to 64 on unpacking.
|
||||
sigbits := 64 - leading - trailing
|
||||
a.b.writeBits(uint64(sigbits), 6)
|
||||
a.b.writeBits(vDelta>>trailing, int(sigbits))
|
||||
}
|
||||
}
|
||||
|
||||
type xorIterator struct {
|
||||
br bstream
|
||||
numTotal uint16
|
||||
numRead uint16
|
||||
|
||||
t int64
|
||||
val float64
|
||||
|
||||
leading uint8
|
||||
trailing uint8
|
||||
|
||||
tDelta uint64
|
||||
err error
|
||||
}
|
||||
|
||||
func (it *xorIterator) At() (int64, float64) {
|
||||
return it.t, it.val
|
||||
}
|
||||
|
||||
func (it *xorIterator) Err() error {
|
||||
return it.err
|
||||
}
|
||||
|
||||
func (it *xorIterator) Next() bool {
|
||||
if it.err != nil || it.numRead == it.numTotal {
|
||||
return false
|
||||
}
|
||||
|
||||
if it.numRead == 0 {
|
||||
t, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
v, err := it.br.readBits(64)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.t = t
|
||||
it.val = math.Float64frombits(v)
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
if it.numRead == 1 {
|
||||
tDelta, err := binary.ReadUvarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.tDelta = tDelta
|
||||
it.t = it.t + int64(it.tDelta)
|
||||
|
||||
return it.readValue()
|
||||
}
|
||||
|
||||
var d byte
|
||||
// read delta-of-delta
|
||||
for i := 0; i < 4; i++ {
|
||||
d <<= 1
|
||||
bit, err := it.br.readBit()
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
if bit == zero {
|
||||
break
|
||||
}
|
||||
d |= 1
|
||||
}
|
||||
var sz uint8
|
||||
var dod int64
|
||||
switch d {
|
||||
case 0x00:
|
||||
// dod == 0
|
||||
case 0x02:
|
||||
sz = 14
|
||||
case 0x06:
|
||||
sz = 17
|
||||
case 0x0e:
|
||||
sz = 20
|
||||
case 0x0f:
|
||||
bits, err := it.br.readBits(64)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
dod = int64(bits)
|
||||
}
|
||||
|
||||
if sz != 0 {
|
||||
bits, err := it.br.readBits(int(sz))
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
if bits > (1 << (sz - 1)) {
|
||||
// or something
|
||||
bits = bits - (1 << sz)
|
||||
}
|
||||
dod = int64(bits)
|
||||
}
|
||||
|
||||
it.tDelta = uint64(int64(it.tDelta) + dod)
|
||||
it.t = it.t + int64(it.tDelta)
|
||||
|
||||
return it.readValue()
|
||||
}
|
||||
|
||||
func (it *xorIterator) readValue() bool {
|
||||
bit, err := it.br.readBit()
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
if bit == zero {
|
||||
// it.val = it.val
|
||||
} else {
|
||||
bit, err := it.br.readBit()
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
if bit == zero {
|
||||
// reuse leading/trailing zero bits
|
||||
// it.leading, it.trailing = it.leading, it.trailing
|
||||
} else {
|
||||
bits, err := it.br.readBits(5)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.leading = uint8(bits)
|
||||
|
||||
bits, err = it.br.readBits(6)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
mbits := uint8(bits)
|
||||
// 0 significant bits here means we overflowed and we actually need 64; see comment in encoder
|
||||
if mbits == 0 {
|
||||
mbits = 64
|
||||
}
|
||||
it.trailing = 64 - it.leading - mbits
|
||||
}
|
||||
|
||||
mbits := int(64 - it.leading - it.trailing)
|
||||
bits, err := it.br.readBits(mbits)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
vbits := math.Float64bits(it.val)
|
||||
vbits ^= (bits << it.trailing)
|
||||
it.val = math.Float64frombits(vbits)
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
499
vendor/github.com/prometheus/tsdb/chunks/chunks.go
generated
vendored
Normal file
499
vendor/github.com/prometheus/tsdb/chunks/chunks.go
generated
vendored
Normal file
|
@ -0,0 +1,499 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chunks
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/tsdb/fileutil"
|
||||
)
|
||||
|
||||
const (
|
||||
// MagicChunks is 4 bytes at the head of a series file.
|
||||
MagicChunks = 0x85BD40DD
|
||||
)
|
||||
|
||||
// Meta holds information about a chunk of data.
|
||||
type Meta struct {
|
||||
// Ref and Chunk hold either a reference that can be used to retrieve
|
||||
// chunk data or the data itself.
|
||||
// Generally, only one of them is set.
|
||||
Ref uint64
|
||||
Chunk chunkenc.Chunk
|
||||
|
||||
MinTime, MaxTime int64 // time range the data covers
|
||||
}
|
||||
|
||||
// writeHash writes the chunk encoding and raw data into the provided hash.
|
||||
func (cm *Meta) writeHash(h hash.Hash) error {
|
||||
if _, err := h.Write([]byte{byte(cm.Chunk.Encoding())}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := h.Write(cm.Chunk.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Returns true if the chunk overlaps [mint, maxt].
|
||||
func (cm *Meta) OverlapsClosedInterval(mint, maxt int64) bool {
|
||||
// The chunk itself is a closed interval [cm.MinTime, cm.MaxTime].
|
||||
return cm.MinTime <= maxt && mint <= cm.MaxTime
|
||||
}
|
||||
|
||||
var (
|
||||
errInvalidSize = fmt.Errorf("invalid size")
|
||||
)
|
||||
|
||||
var castagnoliTable *crc32.Table
|
||||
|
||||
func init() {
|
||||
castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
}
|
||||
|
||||
// newCRC32 initializes a CRC32 hash with a preconfigured polynomial, so the
|
||||
// polynomial may be easily changed in one location at a later time, if necessary.
|
||||
func newCRC32() hash.Hash32 {
|
||||
return crc32.New(castagnoliTable)
|
||||
}
|
||||
|
||||
// Writer implements the ChunkWriter interface for the standard
|
||||
// serialization format.
|
||||
type Writer struct {
|
||||
dirFile *os.File
|
||||
files []*os.File
|
||||
wbuf *bufio.Writer
|
||||
n int64
|
||||
crc32 hash.Hash
|
||||
|
||||
segmentSize int64
|
||||
}
|
||||
|
||||
const (
|
||||
defaultChunkSegmentSize = 512 * 1024 * 1024
|
||||
|
||||
chunksFormatV1 = 1
|
||||
)
|
||||
|
||||
// NewWriter returns a new writer against the given directory.
|
||||
func NewWriter(dir string) (*Writer, error) {
|
||||
if err := os.MkdirAll(dir, 0777); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dirFile, err := fileutil.OpenDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cw := &Writer{
|
||||
dirFile: dirFile,
|
||||
n: 0,
|
||||
crc32: newCRC32(),
|
||||
segmentSize: defaultChunkSegmentSize,
|
||||
}
|
||||
return cw, nil
|
||||
}
|
||||
|
||||
func (w *Writer) tail() *os.File {
|
||||
if len(w.files) == 0 {
|
||||
return nil
|
||||
}
|
||||
return w.files[len(w.files)-1]
|
||||
}
|
||||
|
||||
// finalizeTail writes all pending data to the current tail file,
|
||||
// truncates its size, and closes it.
|
||||
func (w *Writer) finalizeTail() error {
|
||||
tf := w.tail()
|
||||
if tf == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := w.wbuf.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fileutil.Fsync(tf); err != nil {
|
||||
return err
|
||||
}
|
||||
// As the file was pre-allocated, we truncate any superfluous zero bytes.
|
||||
off, err := tf.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tf.Truncate(off); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tf.Close()
|
||||
}
|
||||
|
||||
func (w *Writer) cut() error {
|
||||
// Sync current tail to disk and close.
|
||||
if err := w.finalizeTail(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p, _, err := nextSequenceFile(w.dirFile.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = fileutil.Preallocate(f, w.segmentSize, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = w.dirFile.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write header metadata for new file.
|
||||
|
||||
metab := make([]byte, 8)
|
||||
binary.BigEndian.PutUint32(metab[:4], MagicChunks)
|
||||
metab[4] = chunksFormatV1
|
||||
|
||||
if _, err := f.Write(metab); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.files = append(w.files, f)
|
||||
if w.wbuf != nil {
|
||||
w.wbuf.Reset(f)
|
||||
} else {
|
||||
w.wbuf = bufio.NewWriterSize(f, 8*1024*1024)
|
||||
}
|
||||
w.n = 8
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) write(b []byte) error {
|
||||
n, err := w.wbuf.Write(b)
|
||||
w.n += int64(n)
|
||||
return err
|
||||
}
|
||||
|
||||
// MergeOverlappingChunks removes the samples whose timestamp is overlapping.
|
||||
// The last appearing sample is retained in case there is overlapping.
|
||||
// This assumes that `chks []Meta` is sorted w.r.t. MinTime.
|
||||
func MergeOverlappingChunks(chks []Meta) ([]Meta, error) {
|
||||
if len(chks) < 2 {
|
||||
return chks, nil
|
||||
}
|
||||
newChks := make([]Meta, 0, len(chks)) // Will contain the merged chunks.
|
||||
newChks = append(newChks, chks[0])
|
||||
last := 0
|
||||
for _, c := range chks[1:] {
|
||||
// We need to check only the last chunk in newChks.
|
||||
// Reason: (1) newChks[last-1].MaxTime < newChks[last].MinTime (non overlapping)
|
||||
// (2) As chks are sorted w.r.t. MinTime, newChks[last].MinTime < c.MinTime.
|
||||
// So never overlaps with newChks[last-1] or anything before that.
|
||||
if c.MinTime > newChks[last].MaxTime {
|
||||
newChks = append(newChks, c)
|
||||
last += 1
|
||||
continue
|
||||
}
|
||||
nc := &newChks[last]
|
||||
if c.MaxTime > nc.MaxTime {
|
||||
nc.MaxTime = c.MaxTime
|
||||
}
|
||||
chk, err := MergeChunks(nc.Chunk, c.Chunk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nc.Chunk = chk
|
||||
}
|
||||
|
||||
return newChks, nil
|
||||
}
|
||||
|
||||
// MergeChunks vertically merges a and b, i.e., if there is any sample
|
||||
// with same timestamp in both a and b, the sample in a is discarded.
|
||||
func MergeChunks(a, b chunkenc.Chunk) (*chunkenc.XORChunk, error) {
|
||||
newChunk := chunkenc.NewXORChunk()
|
||||
app, err := newChunk.Appender()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ait := a.Iterator()
|
||||
bit := b.Iterator()
|
||||
aok, bok := ait.Next(), bit.Next()
|
||||
for aok && bok {
|
||||
at, av := ait.At()
|
||||
bt, bv := bit.At()
|
||||
if at < bt {
|
||||
app.Append(at, av)
|
||||
aok = ait.Next()
|
||||
} else if bt < at {
|
||||
app.Append(bt, bv)
|
||||
bok = bit.Next()
|
||||
} else {
|
||||
app.Append(bt, bv)
|
||||
aok = ait.Next()
|
||||
bok = bit.Next()
|
||||
}
|
||||
}
|
||||
for aok {
|
||||
at, av := ait.At()
|
||||
app.Append(at, av)
|
||||
aok = ait.Next()
|
||||
}
|
||||
for bok {
|
||||
bt, bv := bit.At()
|
||||
app.Append(bt, bv)
|
||||
bok = bit.Next()
|
||||
}
|
||||
if ait.Err() != nil {
|
||||
return nil, ait.Err()
|
||||
}
|
||||
if bit.Err() != nil {
|
||||
return nil, bit.Err()
|
||||
}
|
||||
return newChunk, nil
|
||||
}
|
||||
|
||||
func (w *Writer) WriteChunks(chks ...Meta) error {
|
||||
// Calculate maximum space we need and cut a new segment in case
|
||||
// we don't fit into the current one.
|
||||
maxLen := int64(binary.MaxVarintLen32) // The number of chunks.
|
||||
for _, c := range chks {
|
||||
maxLen += binary.MaxVarintLen32 + 1 // The number of bytes in the chunk and its encoding.
|
||||
maxLen += int64(len(c.Chunk.Bytes()))
|
||||
maxLen += 4 // The 4 bytes of crc32
|
||||
}
|
||||
newsz := w.n + maxLen
|
||||
|
||||
if w.wbuf == nil || w.n > w.segmentSize || newsz > w.segmentSize && maxLen <= w.segmentSize {
|
||||
if err := w.cut(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
b = [binary.MaxVarintLen32]byte{}
|
||||
seq = uint64(w.seq()) << 32
|
||||
)
|
||||
for i := range chks {
|
||||
chk := &chks[i]
|
||||
|
||||
chk.Ref = seq | uint64(w.n)
|
||||
|
||||
n := binary.PutUvarint(b[:], uint64(len(chk.Chunk.Bytes())))
|
||||
|
||||
if err := w.write(b[:n]); err != nil {
|
||||
return err
|
||||
}
|
||||
b[0] = byte(chk.Chunk.Encoding())
|
||||
if err := w.write(b[:1]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.write(chk.Chunk.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.crc32.Reset()
|
||||
if err := chk.writeHash(w.crc32); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.write(w.crc32.Sum(b[:0])); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) seq() int {
|
||||
return len(w.files) - 1
|
||||
}
|
||||
|
||||
func (w *Writer) Close() error {
|
||||
if err := w.finalizeTail(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// close dir file (if not windows platform will fail on rename)
|
||||
return w.dirFile.Close()
|
||||
}
|
||||
|
||||
// ByteSlice abstracts a byte slice.
|
||||
type ByteSlice interface {
|
||||
Len() int
|
||||
Range(start, end int) []byte
|
||||
}
|
||||
|
||||
type realByteSlice []byte
|
||||
|
||||
func (b realByteSlice) Len() int {
|
||||
return len(b)
|
||||
}
|
||||
|
||||
func (b realByteSlice) Range(start, end int) []byte {
|
||||
return b[start:end]
|
||||
}
|
||||
|
||||
func (b realByteSlice) Sub(start, end int) ByteSlice {
|
||||
return b[start:end]
|
||||
}
|
||||
|
||||
// Reader implements a SeriesReader for a serialized byte stream
|
||||
// of series data.
|
||||
type Reader struct {
|
||||
bs []ByteSlice // The underlying bytes holding the encoded series data.
|
||||
cs []io.Closer // Closers for resources behind the byte slices.
|
||||
size int64 // The total size of bytes in the reader.
|
||||
pool chunkenc.Pool
|
||||
}
|
||||
|
||||
func newReader(bs []ByteSlice, cs []io.Closer, pool chunkenc.Pool) (*Reader, error) {
|
||||
cr := Reader{pool: pool, bs: bs, cs: cs}
|
||||
var totalSize int64
|
||||
|
||||
for i, b := range cr.bs {
|
||||
if b.Len() < 4 {
|
||||
return nil, errors.Wrapf(errInvalidSize, "validate magic in segment %d", i)
|
||||
}
|
||||
// Verify magic number.
|
||||
if m := binary.BigEndian.Uint32(b.Range(0, 4)); m != MagicChunks {
|
||||
return nil, errors.Errorf("invalid magic number %x", m)
|
||||
}
|
||||
totalSize += int64(b.Len())
|
||||
}
|
||||
cr.size = totalSize
|
||||
return &cr, nil
|
||||
}
|
||||
|
||||
// NewReader returns a new chunk reader against the given byte slices.
|
||||
func NewReader(bs []ByteSlice, pool chunkenc.Pool) (*Reader, error) {
|
||||
if pool == nil {
|
||||
pool = chunkenc.NewPool()
|
||||
}
|
||||
return newReader(bs, nil, pool)
|
||||
}
|
||||
|
||||
// NewDirReader returns a new Reader against sequentially numbered files in the
|
||||
// given directory.
|
||||
func NewDirReader(dir string, pool chunkenc.Pool) (*Reader, error) {
|
||||
files, err := sequenceFiles(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if pool == nil {
|
||||
pool = chunkenc.NewPool()
|
||||
}
|
||||
|
||||
var (
|
||||
bs []ByteSlice
|
||||
cs []io.Closer
|
||||
)
|
||||
for _, fn := range files {
|
||||
f, err := fileutil.OpenMmapFile(fn)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "mmap files")
|
||||
}
|
||||
cs = append(cs, f)
|
||||
bs = append(bs, realByteSlice(f.Bytes()))
|
||||
}
|
||||
return newReader(bs, cs, pool)
|
||||
}
|
||||
|
||||
func (s *Reader) Close() error {
|
||||
return closeAll(s.cs)
|
||||
}
|
||||
|
||||
// Size returns the size of the chunks.
|
||||
func (s *Reader) Size() int64 {
|
||||
return s.size
|
||||
}
|
||||
|
||||
// Chunk returns a chunk from a given reference.
|
||||
func (s *Reader) Chunk(ref uint64) (chunkenc.Chunk, error) {
|
||||
var (
|
||||
sgmSeq = int(ref >> 32)
|
||||
sgmOffset = int((ref << 32) >> 32)
|
||||
)
|
||||
if sgmSeq >= len(s.bs) {
|
||||
return nil, errors.Errorf("reference sequence %d out of range", sgmSeq)
|
||||
}
|
||||
chkS := s.bs[sgmSeq]
|
||||
|
||||
if sgmOffset >= chkS.Len() {
|
||||
return nil, errors.Errorf("offset %d beyond data size %d", sgmOffset, chkS.Len())
|
||||
}
|
||||
// With the minimum chunk length this should never cause us reading
|
||||
// over the end of the slice.
|
||||
chk := chkS.Range(sgmOffset, sgmOffset+binary.MaxVarintLen32)
|
||||
|
||||
chkLen, n := binary.Uvarint(chk)
|
||||
if n <= 0 {
|
||||
return nil, errors.Errorf("reading chunk length failed with %d", n)
|
||||
}
|
||||
chk = chkS.Range(sgmOffset+n, sgmOffset+n+1+int(chkLen))
|
||||
|
||||
return s.pool.Get(chunkenc.Encoding(chk[0]), chk[1:1+chkLen])
|
||||
}
|
||||
|
||||
func nextSequenceFile(dir string) (string, int, error) {
|
||||
names, err := fileutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
i := uint64(0)
|
||||
for _, n := range names {
|
||||
j, err := strconv.ParseUint(n, 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
i = j
|
||||
}
|
||||
return filepath.Join(dir, fmt.Sprintf("%0.6d", i+1)), int(i + 1), nil
|
||||
}
|
||||
|
||||
func sequenceFiles(dir string) ([]string, error) {
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var res []string
|
||||
|
||||
for _, fi := range files {
|
||||
if _, err := strconv.ParseUint(fi.Name(), 10, 64); err != nil {
|
||||
continue
|
||||
}
|
||||
res = append(res, filepath.Join(dir, fi.Name()))
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func closeAll(cs []io.Closer) (err error) {
|
||||
for _, c := range cs {
|
||||
if e := c.Close(); e != nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
1033
vendor/github.com/prometheus/tsdb/compact.go
generated
vendored
Normal file
1033
vendor/github.com/prometheus/tsdb/compact.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load diff
1152
vendor/github.com/prometheus/tsdb/db.go
generated
vendored
Normal file
1152
vendor/github.com/prometheus/tsdb/db.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load diff
244
vendor/github.com/prometheus/tsdb/encoding/encoding.go
generated
vendored
Normal file
244
vendor/github.com/prometheus/tsdb/encoding/encoding.go
generated
vendored
Normal file
|
@ -0,0 +1,244 @@
|
|||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package encoding
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
"unsafe"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidSize = errors.New("invalid size")
|
||||
ErrInvalidChecksum = errors.New("invalid checksum")
|
||||
)
|
||||
|
||||
// enbuf is a helper type to populate a byte slice with various types.
|
||||
type Encbuf struct {
|
||||
B []byte
|
||||
C [binary.MaxVarintLen64]byte
|
||||
}
|
||||
|
||||
func (e *Encbuf) Reset() { e.B = e.B[:0] }
|
||||
func (e *Encbuf) Get() []byte { return e.B }
|
||||
func (e *Encbuf) Len() int { return len(e.B) }
|
||||
|
||||
func (e *Encbuf) PutString(s string) { e.B = append(e.B, s...) }
|
||||
func (e *Encbuf) PutByte(c byte) { e.B = append(e.B, c) }
|
||||
|
||||
func (e *Encbuf) PutBE32int(x int) { e.PutBE32(uint32(x)) }
|
||||
func (e *Encbuf) PutUvarint32(x uint32) { e.PutUvarint64(uint64(x)) }
|
||||
func (e *Encbuf) PutBE64int64(x int64) { e.PutBE64(uint64(x)) }
|
||||
func (e *Encbuf) PutUvarint(x int) { e.PutUvarint64(uint64(x)) }
|
||||
|
||||
func (e *Encbuf) PutBE32(x uint32) {
|
||||
binary.BigEndian.PutUint32(e.C[:], x)
|
||||
e.B = append(e.B, e.C[:4]...)
|
||||
}
|
||||
|
||||
func (e *Encbuf) PutBE64(x uint64) {
|
||||
binary.BigEndian.PutUint64(e.C[:], x)
|
||||
e.B = append(e.B, e.C[:8]...)
|
||||
}
|
||||
|
||||
func (e *Encbuf) PutUvarint64(x uint64) {
|
||||
n := binary.PutUvarint(e.C[:], x)
|
||||
e.B = append(e.B, e.C[:n]...)
|
||||
}
|
||||
|
||||
func (e *Encbuf) PutVarint64(x int64) {
|
||||
n := binary.PutVarint(e.C[:], x)
|
||||
e.B = append(e.B, e.C[:n]...)
|
||||
}
|
||||
|
||||
// putVarintStr writes a string to the buffer prefixed by its varint length (in bytes!).
|
||||
func (e *Encbuf) PutUvarintStr(s string) {
|
||||
b := *(*[]byte)(unsafe.Pointer(&s))
|
||||
e.PutUvarint(len(b))
|
||||
e.PutString(s)
|
||||
}
|
||||
|
||||
// putHash appends a hash over the buffers current contents to the buffer.
|
||||
func (e *Encbuf) PutHash(h hash.Hash) {
|
||||
h.Reset()
|
||||
_, err := h.Write(e.B)
|
||||
if err != nil {
|
||||
panic(err) // The CRC32 implementation does not error
|
||||
}
|
||||
e.B = h.Sum(e.B)
|
||||
}
|
||||
|
||||
// decbuf provides safe methods to extract data from a byte slice. It does all
|
||||
// necessary bounds checking and advancing of the byte slice.
|
||||
// Several datums can be extracted without checking for errors. However, before using
|
||||
// any datum, the err() method must be checked.
|
||||
type Decbuf struct {
|
||||
B []byte
|
||||
E error
|
||||
}
|
||||
|
||||
// NewDecbufAt returns a new decoding buffer. It expects the first 4 bytes
|
||||
// after offset to hold the big endian encoded content length, followed by the contents and the expected
|
||||
// checksum.
|
||||
func NewDecbufAt(bs ByteSlice, off int, castagnoliTable *crc32.Table) Decbuf {
|
||||
if bs.Len() < off+4 {
|
||||
return Decbuf{E: ErrInvalidSize}
|
||||
}
|
||||
b := bs.Range(off, off+4)
|
||||
l := int(binary.BigEndian.Uint32(b))
|
||||
|
||||
if bs.Len() < off+4+l+4 {
|
||||
return Decbuf{E: ErrInvalidSize}
|
||||
}
|
||||
|
||||
// Load bytes holding the contents plus a CRC32 checksum.
|
||||
b = bs.Range(off+4, off+4+l+4)
|
||||
dec := Decbuf{B: b[:len(b)-4]}
|
||||
|
||||
if exp := binary.BigEndian.Uint32(b[len(b)-4:]); dec.Crc32(castagnoliTable) != exp {
|
||||
return Decbuf{E: ErrInvalidChecksum}
|
||||
}
|
||||
return dec
|
||||
}
|
||||
|
||||
// NewDecbufUvarintAt returns a new decoding buffer. It expects the first bytes
|
||||
// after offset to hold the uvarint-encoded buffers length, followed by the contents and the expected
|
||||
// checksum.
|
||||
func NewDecbufUvarintAt(bs ByteSlice, off int, castagnoliTable *crc32.Table) Decbuf {
|
||||
// We never have to access this method at the far end of the byte slice. Thus just checking
|
||||
// against the MaxVarintLen32 is sufficient.
|
||||
if bs.Len() < off+binary.MaxVarintLen32 {
|
||||
return Decbuf{E: ErrInvalidSize}
|
||||
}
|
||||
b := bs.Range(off, off+binary.MaxVarintLen32)
|
||||
|
||||
l, n := binary.Uvarint(b)
|
||||
if n <= 0 || n > binary.MaxVarintLen32 {
|
||||
return Decbuf{E: errors.Errorf("invalid uvarint %d", n)}
|
||||
}
|
||||
|
||||
if bs.Len() < off+n+int(l)+4 {
|
||||
return Decbuf{E: ErrInvalidSize}
|
||||
}
|
||||
|
||||
// Load bytes holding the contents plus a CRC32 checksum.
|
||||
b = bs.Range(off+n, off+n+int(l)+4)
|
||||
dec := Decbuf{B: b[:len(b)-4]}
|
||||
|
||||
if dec.Crc32(castagnoliTable) != binary.BigEndian.Uint32(b[len(b)-4:]) {
|
||||
return Decbuf{E: ErrInvalidChecksum}
|
||||
}
|
||||
return dec
|
||||
}
|
||||
|
||||
func (d *Decbuf) Uvarint() int { return int(d.Uvarint64()) }
|
||||
func (d *Decbuf) Be32int() int { return int(d.Be32()) }
|
||||
func (d *Decbuf) Be64int64() int64 { return int64(d.Be64()) }
|
||||
|
||||
// Crc32 returns a CRC32 checksum over the remaining bytes.
|
||||
func (d *Decbuf) Crc32(castagnoliTable *crc32.Table) uint32 {
|
||||
return crc32.Checksum(d.B, castagnoliTable)
|
||||
}
|
||||
|
||||
func (d *Decbuf) UvarintStr() string {
|
||||
l := d.Uvarint64()
|
||||
if d.E != nil {
|
||||
return ""
|
||||
}
|
||||
if len(d.B) < int(l) {
|
||||
d.E = ErrInvalidSize
|
||||
return ""
|
||||
}
|
||||
s := string(d.B[:l])
|
||||
d.B = d.B[l:]
|
||||
return s
|
||||
}
|
||||
|
||||
func (d *Decbuf) Varint64() int64 {
|
||||
if d.E != nil {
|
||||
return 0
|
||||
}
|
||||
x, n := binary.Varint(d.B)
|
||||
if n < 1 {
|
||||
d.E = ErrInvalidSize
|
||||
return 0
|
||||
}
|
||||
d.B = d.B[n:]
|
||||
return x
|
||||
}
|
||||
|
||||
func (d *Decbuf) Uvarint64() uint64 {
|
||||
if d.E != nil {
|
||||
return 0
|
||||
}
|
||||
x, n := binary.Uvarint(d.B)
|
||||
if n < 1 {
|
||||
d.E = ErrInvalidSize
|
||||
return 0
|
||||
}
|
||||
d.B = d.B[n:]
|
||||
return x
|
||||
}
|
||||
|
||||
func (d *Decbuf) Be64() uint64 {
|
||||
if d.E != nil {
|
||||
return 0
|
||||
}
|
||||
if len(d.B) < 8 {
|
||||
d.E = ErrInvalidSize
|
||||
return 0
|
||||
}
|
||||
x := binary.BigEndian.Uint64(d.B)
|
||||
d.B = d.B[8:]
|
||||
return x
|
||||
}
|
||||
|
||||
func (d *Decbuf) Be32() uint32 {
|
||||
if d.E != nil {
|
||||
return 0
|
||||
}
|
||||
if len(d.B) < 4 {
|
||||
d.E = ErrInvalidSize
|
||||
return 0
|
||||
}
|
||||
x := binary.BigEndian.Uint32(d.B)
|
||||
d.B = d.B[4:]
|
||||
return x
|
||||
}
|
||||
|
||||
func (d *Decbuf) Byte() byte {
|
||||
if d.E != nil {
|
||||
return 0
|
||||
}
|
||||
if len(d.B) < 1 {
|
||||
d.E = ErrInvalidSize
|
||||
return 0
|
||||
}
|
||||
x := d.B[0]
|
||||
d.B = d.B[1:]
|
||||
return x
|
||||
}
|
||||
|
||||
func (d *Decbuf) Err() error { return d.E }
|
||||
func (d *Decbuf) Len() int { return len(d.B) }
|
||||
func (d *Decbuf) Get() []byte { return d.B }
|
||||
|
||||
// ByteSlice abstracts a byte slice.
|
||||
type ByteSlice interface {
|
||||
Len() int
|
||||
Range(start, end int) []byte
|
||||
}
|
22
vendor/github.com/prometheus/tsdb/fileutil/dir_unix.go
generated
vendored
Normal file
22
vendor/github.com/prometheus/tsdb/fileutil/dir_unix.go
generated
vendored
Normal file
|
@ -0,0 +1,22 @@
|
|||
// Copyright 2016 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build !windows
|
||||
|
||||
package fileutil
|
||||
|
||||
import "os"
|
||||
|
||||
// OpenDir opens a directory for syncing.
|
||||
func OpenDir(path string) (*os.File, error) { return os.Open(path) }
|
46
vendor/github.com/prometheus/tsdb/fileutil/dir_windows.go
generated
vendored
Normal file
46
vendor/github.com/prometheus/tsdb/fileutil/dir_windows.go
generated
vendored
Normal file
|
@ -0,0 +1,46 @@
|
|||
// Copyright 2016 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build windows
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// OpenDir opens a directory in windows with write access for syncing.
|
||||
func OpenDir(path string) (*os.File, error) {
|
||||
fd, err := openDir(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return os.NewFile(uintptr(fd), path), nil
|
||||
}
|
||||
|
||||
func openDir(path string) (fd syscall.Handle, err error) {
|
||||
if len(path) == 0 {
|
||||
return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND
|
||||
}
|
||||
pathp, err := syscall.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return syscall.InvalidHandle, err
|
||||
}
|
||||
access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE)
|
||||
sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE)
|
||||
createmode := uint32(syscall.OPEN_EXISTING)
|
||||
fl := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS)
|
||||
return syscall.CreateFile(pathp, access, sharemode, nil, createmode, fl, 0)
|
||||
}
|
149
vendor/github.com/prometheus/tsdb/fileutil/fileutil.go
generated
vendored
Normal file
149
vendor/github.com/prometheus/tsdb/fileutil/fileutil.go
generated
vendored
Normal file
|
@ -0,0 +1,149 @@
|
|||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package fileutil provides utility methods used when dealing with the filesystem in tsdb.
|
||||
// It is largely copied from github.com/coreos/etcd/pkg/fileutil to avoid the
|
||||
// dependency chain it brings with it.
|
||||
// Please check github.com/coreos/etcd for licensing information.
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CopyDirs copies all directories, subdirectories and files recursively including the empty folders.
|
||||
// Source and destination must be full paths.
|
||||
func CopyDirs(src, dest string) error {
|
||||
if err := os.MkdirAll(dest, 0777); err != nil {
|
||||
return err
|
||||
}
|
||||
files, err := readDirs(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
dp := filepath.Join(dest, f)
|
||||
sp := filepath.Join(src, f)
|
||||
|
||||
stat, err := os.Stat(sp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Empty directories are also created.
|
||||
if stat.IsDir() {
|
||||
if err := os.MkdirAll(dp, 0777); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if err := copyFile(sp, dp); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyFile(src, dest string) error {
|
||||
data, err := ioutil.ReadFile(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(dest, data, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readDirs reads the source directory recursively and
|
||||
// returns relative paths to all files and empty directories.
|
||||
func readDirs(src string) ([]string, error) {
|
||||
var files []string
|
||||
|
||||
err := filepath.Walk(src, func(path string, f os.FileInfo, err error) error {
|
||||
relativePath := strings.TrimPrefix(path, src)
|
||||
if len(relativePath) > 0 {
|
||||
files = append(files, relativePath)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// ReadDir returns the filenames in the given directory in sorted order.
|
||||
func ReadDir(dirpath string) ([]string, error) {
|
||||
dir, err := os.Open(dirpath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer dir.Close()
|
||||
names, err := dir.Readdirnames(-1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// Rename safely renames a file.
|
||||
func Rename(from, to string) error {
|
||||
if err := os.Rename(from, to); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Directory was renamed; sync parent dir to persist rename.
|
||||
pdir, err := OpenDir(filepath.Dir(to))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = Fsync(pdir); err != nil {
|
||||
pdir.Close()
|
||||
return err
|
||||
}
|
||||
return pdir.Close()
|
||||
}
|
||||
|
||||
// Replace moves a file or directory to a new location and deletes any previous data.
|
||||
// It is not atomic.
|
||||
func Replace(from, to string) error {
|
||||
if err := os.RemoveAll(to); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(from, to); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Directory was renamed; sync parent dir to persist rename.
|
||||
pdir, err := OpenDir(filepath.Dir(to))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = Fsync(pdir); err != nil {
|
||||
pdir.Close()
|
||||
return err
|
||||
}
|
||||
return pdir.Close()
|
||||
}
|
41
vendor/github.com/prometheus/tsdb/fileutil/flock.go
generated
vendored
Normal file
41
vendor/github.com/prometheus/tsdb/fileutil/flock.go
generated
vendored
Normal file
|
@ -0,0 +1,41 @@
|
|||
// Copyright 2016 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// Releaser provides the Release method to release a file lock.
|
||||
type Releaser interface {
|
||||
Release() error
|
||||
}
|
||||
|
||||
// Flock locks the file with the provided name. If the file does not exist, it is
|
||||
// created. The returned Releaser is used to release the lock. existed is true
|
||||
// if the file to lock already existed. A non-nil error is returned if the
|
||||
// locking has failed. Neither this function nor the returned Releaser is
|
||||
// goroutine-safe.
|
||||
func Flock(fileName string) (r Releaser, existed bool, err error) {
|
||||
if err = os.MkdirAll(filepath.Dir(fileName), 0755); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
_, err = os.Stat(fileName)
|
||||
existed = err == nil
|
||||
|
||||
r, err = newLock(fileName)
|
||||
return r, existed, err
|
||||
}
|
32
vendor/github.com/prometheus/tsdb/fileutil/flock_plan9.go
generated
vendored
Normal file
32
vendor/github.com/prometheus/tsdb/fileutil/flock_plan9.go
generated
vendored
Normal file
|
@ -0,0 +1,32 @@
|
|||
// Copyright 2016 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fileutil
|
||||
|
||||
import "os"
|
||||
|
||||
type plan9Lock struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (l *plan9Lock) Release() error {
|
||||
return l.f.Close()
|
||||
}
|
||||
|
||||
func newLock(fileName string) (Releaser, error) {
|
||||
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &plan9Lock{f}, nil
|
||||
}
|
59
vendor/github.com/prometheus/tsdb/fileutil/flock_solaris.go
generated
vendored
Normal file
59
vendor/github.com/prometheus/tsdb/fileutil/flock_solaris.go
generated
vendored
Normal file
|
@ -0,0 +1,59 @@
|
|||
// Copyright 2016 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build solaris
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type unixLock struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (l *unixLock) Release() error {
|
||||
if err := l.set(false); err != nil {
|
||||
return err
|
||||
}
|
||||
return l.f.Close()
|
||||
}
|
||||
|
||||
func (l *unixLock) set(lock bool) error {
|
||||
flock := syscall.Flock_t{
|
||||
Type: syscall.F_UNLCK,
|
||||
Start: 0,
|
||||
Len: 0,
|
||||
Whence: 1,
|
||||
}
|
||||
if lock {
|
||||
flock.Type = syscall.F_WRLCK
|
||||
}
|
||||
return syscall.FcntlFlock(l.f.Fd(), syscall.F_SETLK, &flock)
|
||||
}
|
||||
|
||||
func newLock(fileName string) (Releaser, error) {
|
||||
f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l := &unixLock{f}
|
||||
err = l.set(true)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
return l, nil
|
||||
}
|
54
vendor/github.com/prometheus/tsdb/fileutil/flock_unix.go
generated
vendored
Normal file
54
vendor/github.com/prometheus/tsdb/fileutil/flock_unix.go
generated
vendored
Normal file
|
@ -0,0 +1,54 @@
|
|||
// Copyright 2016 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build darwin dragonfly freebsd linux netbsd openbsd
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type unixLock struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (l *unixLock) Release() error {
|
||||
if err := l.set(false); err != nil {
|
||||
return err
|
||||
}
|
||||
return l.f.Close()
|
||||
}
|
||||
|
||||
func (l *unixLock) set(lock bool) error {
|
||||
how := syscall.LOCK_UN
|
||||
if lock {
|
||||
how = syscall.LOCK_EX
|
||||
}
|
||||
return syscall.Flock(int(l.f.Fd()), how|syscall.LOCK_NB)
|
||||
}
|
||||
|
||||
func newLock(fileName string) (Releaser, error) {
|
||||
f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l := &unixLock{f}
|
||||
err = l.set(true)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
return l, nil
|
||||
}
|
36
vendor/github.com/prometheus/tsdb/fileutil/flock_windows.go
generated
vendored
Normal file
36
vendor/github.com/prometheus/tsdb/fileutil/flock_windows.go
generated
vendored
Normal file
|
@ -0,0 +1,36 @@
|
|||
// Copyright 2016 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fileutil
|
||||
|
||||
import "syscall"
|
||||
|
||||
type windowsLock struct {
|
||||
fd syscall.Handle
|
||||
}
|
||||
|
||||
func (fl *windowsLock) Release() error {
|
||||
return syscall.Close(fl.fd)
|
||||
}
|
||||
|
||||
func newLock(fileName string) (Releaser, error) {
|
||||
pathp, err := syscall.UTF16PtrFromString(fileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fd, err := syscall.CreateFile(pathp, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, syscall.CREATE_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &windowsLock{fd}, nil
|
||||
}
|
61
vendor/github.com/prometheus/tsdb/fileutil/mmap.go
generated
vendored
Normal file
61
vendor/github.com/prometheus/tsdb/fileutil/mmap.go
generated
vendored
Normal file
|
@ -0,0 +1,61 @@
|
|||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type MmapFile struct {
|
||||
f *os.File
|
||||
b []byte
|
||||
}
|
||||
|
||||
func OpenMmapFile(path string) (*MmapFile, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "try lock file")
|
||||
}
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "stat")
|
||||
}
|
||||
|
||||
b, err := mmap(f, int(info.Size()))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "mmap")
|
||||
}
|
||||
|
||||
return &MmapFile{f: f, b: b}, nil
|
||||
}
|
||||
|
||||
func (f *MmapFile) Close() error {
|
||||
err0 := munmap(f.b)
|
||||
err1 := f.f.Close()
|
||||
|
||||
if err0 != nil {
|
||||
return err0
|
||||
}
|
||||
return err1
|
||||
}
|
||||
|
||||
func (f *MmapFile) File() *os.File {
|
||||
return f.f
|
||||
}
|
||||
|
||||
func (f *MmapFile) Bytes() []byte {
|
||||
return f.b
|
||||
}
|
18
vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go
generated
vendored
Normal file
18
vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go
generated
vendored
Normal file
|
@ -0,0 +1,18 @@
|
|||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build windows
|
||||
|
||||
package fileutil
|
||||
|
||||
const maxMapSize = 0x7FFFFFFF // 2GB
|
18
vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go
generated
vendored
Normal file
18
vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go
generated
vendored
Normal file
|
@ -0,0 +1,18 @@
|
|||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build windows
|
||||
|
||||
package fileutil
|
||||
|
||||
const maxMapSize = 0xFFFFFFFFFFFF // 256TB
|
30
vendor/github.com/prometheus/tsdb/fileutil/mmap_unix.go
generated
vendored
Normal file
30
vendor/github.com/prometheus/tsdb/fileutil/mmap_unix.go
generated
vendored
Normal file
|
@ -0,0 +1,30 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build !windows,!plan9
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func mmap(f *os.File, length int) ([]byte, error) {
|
||||
return unix.Mmap(int(f.Fd()), 0, length, unix.PROT_READ, unix.MAP_SHARED)
|
||||
}
|
||||
|
||||
func munmap(b []byte) (err error) {
|
||||
return unix.Munmap(b)
|
||||
}
|
46
vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go
generated
vendored
Normal file
46
vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go
generated
vendored
Normal file
|
@ -0,0 +1,46 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func mmap(f *os.File, size int) ([]byte, error) {
|
||||
low, high := uint32(size), uint32(size>>32)
|
||||
h, errno := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, high, low, nil)
|
||||
if h == 0 {
|
||||
return nil, os.NewSyscallError("CreateFileMapping", errno)
|
||||
}
|
||||
|
||||
addr, errno := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, uintptr(size))
|
||||
if addr == 0 {
|
||||
return nil, os.NewSyscallError("MapViewOfFile", errno)
|
||||
}
|
||||
|
||||
if err := syscall.CloseHandle(syscall.Handle(h)); err != nil {
|
||||
return nil, os.NewSyscallError("CloseHandle", err)
|
||||
}
|
||||
|
||||
return (*[maxMapSize]byte)(unsafe.Pointer(addr))[:size], nil
|
||||
}
|
||||
|
||||
func munmap(b []byte) error {
|
||||
if err := syscall.UnmapViewOfFile((uintptr)(unsafe.Pointer(&b[0]))); err != nil {
|
||||
return os.NewSyscallError("UnmapViewOfFile", err)
|
||||
}
|
||||
return nil
|
||||
}
|
54
vendor/github.com/prometheus/tsdb/fileutil/preallocate.go
generated
vendored
Normal file
54
vendor/github.com/prometheus/tsdb/fileutil/preallocate.go
generated
vendored
Normal file
|
@ -0,0 +1,54 @@
|
|||
// Copyright 2015 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Preallocate tries to allocate the space for given
|
||||
// file. This operation is only supported on linux by a
|
||||
// few filesystems (btrfs, ext4, etc.).
|
||||
// If the operation is unsupported, no error will be returned.
|
||||
// Otherwise, the error encountered will be returned.
|
||||
func Preallocate(f *os.File, sizeInBytes int64, extendFile bool) error {
|
||||
if sizeInBytes == 0 {
|
||||
// fallocate will return EINVAL if length is 0; skip
|
||||
return nil
|
||||
}
|
||||
if extendFile {
|
||||
return preallocExtend(f, sizeInBytes)
|
||||
}
|
||||
return preallocFixed(f, sizeInBytes)
|
||||
}
|
||||
|
||||
func preallocExtendTrunc(f *os.File, sizeInBytes int64) error {
|
||||
curOff, err := f.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
size, err := f.Seek(sizeInBytes, io.SeekEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = f.Seek(curOff, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
if sizeInBytes > size {
|
||||
return nil
|
||||
}
|
||||
return f.Truncate(sizeInBytes)
|
||||
}
|
41
vendor/github.com/prometheus/tsdb/fileutil/preallocate_darwin.go
generated
vendored
Normal file
41
vendor/github.com/prometheus/tsdb/fileutil/preallocate_darwin.go
generated
vendored
Normal file
|
@ -0,0 +1,41 @@
|
|||
// Copyright 2015 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func preallocExtend(f *os.File, sizeInBytes int64) error {
|
||||
if err := preallocFixed(f, sizeInBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
return preallocExtendTrunc(f, sizeInBytes)
|
||||
}
|
||||
|
||||
func preallocFixed(f *os.File, sizeInBytes int64) error {
|
||||
fstore := &syscall.Fstore_t{
|
||||
Flags: syscall.F_ALLOCATEALL,
|
||||
Posmode: syscall.F_PEOFPOSMODE,
|
||||
Length: sizeInBytes}
|
||||
p := unsafe.Pointer(fstore)
|
||||
_, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_PREALLOCATE), uintptr(p))
|
||||
if errno == 0 || errno == syscall.ENOTSUP {
|
||||
return nil
|
||||
}
|
||||
return errno
|
||||
}
|
47
vendor/github.com/prometheus/tsdb/fileutil/preallocate_linux.go
generated
vendored
Normal file
47
vendor/github.com/prometheus/tsdb/fileutil/preallocate_linux.go
generated
vendored
Normal file
|
@ -0,0 +1,47 @@
|
|||
// Copyright 2015 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func preallocExtend(f *os.File, sizeInBytes int64) error {
|
||||
// use mode = 0 to change size
|
||||
err := syscall.Fallocate(int(f.Fd()), 0, 0, sizeInBytes)
|
||||
if err != nil {
|
||||
errno, ok := err.(syscall.Errno)
|
||||
// not supported; fallback
|
||||
// fallocate EINTRs frequently in some environments; fallback
|
||||
if ok && (errno == syscall.ENOTSUP || errno == syscall.EINTR) {
|
||||
return preallocExtendTrunc(f, sizeInBytes)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func preallocFixed(f *os.File, sizeInBytes int64) error {
|
||||
// use mode = 1 to keep size; see FALLOC_FL_KEEP_SIZE
|
||||
err := syscall.Fallocate(int(f.Fd()), 1, 0, sizeInBytes)
|
||||
if err != nil {
|
||||
errno, ok := err.(syscall.Errno)
|
||||
// treat not supported as nil error
|
||||
if ok && errno == syscall.ENOTSUP {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
25
vendor/github.com/prometheus/tsdb/fileutil/preallocate_other.go
generated
vendored
Normal file
25
vendor/github.com/prometheus/tsdb/fileutil/preallocate_other.go
generated
vendored
Normal file
|
@ -0,0 +1,25 @@
|
|||
// Copyright 2015 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build !linux,!darwin
|
||||
|
||||
package fileutil
|
||||
|
||||
import "os"
|
||||
|
||||
func preallocExtend(f *os.File, sizeInBytes int64) error {
|
||||
return preallocExtendTrunc(f, sizeInBytes)
|
||||
}
|
||||
|
||||
func preallocFixed(f *os.File, sizeInBytes int64) error { return nil }
|
29
vendor/github.com/prometheus/tsdb/fileutil/sync.go
generated
vendored
Normal file
29
vendor/github.com/prometheus/tsdb/fileutil/sync.go
generated
vendored
Normal file
|
@ -0,0 +1,29 @@
|
|||
// Copyright 2016 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build !linux,!darwin
|
||||
|
||||
package fileutil
|
||||
|
||||
import "os"
|
||||
|
||||
// Fsync is a wrapper around file.Sync(). Special handling is needed on darwin platform.
|
||||
func Fsync(f *os.File) error {
|
||||
return f.Sync()
|
||||
}
|
||||
|
||||
// Fdatasync is a wrapper around file.Sync(). Special handling is needed on linux platform.
|
||||
func Fdatasync(f *os.File) error {
|
||||
return f.Sync()
|
||||
}
|
40
vendor/github.com/prometheus/tsdb/fileutil/sync_darwin.go
generated
vendored
Normal file
40
vendor/github.com/prometheus/tsdb/fileutil/sync_darwin.go
generated
vendored
Normal file
|
@ -0,0 +1,40 @@
|
|||
// Copyright 2016 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build darwin
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// Fsync on HFS/OSX flushes the data on to the physical drive but the drive
|
||||
// may not write it to the persistent media for quite sometime and it may be
|
||||
// written in out-of-order sequence. Using F_FULLFSYNC ensures that the
|
||||
// physical drive's buffer will also get flushed to the media.
|
||||
func Fsync(f *os.File) error {
|
||||
_, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_FULLFSYNC), uintptr(0))
|
||||
if errno == 0 {
|
||||
return nil
|
||||
}
|
||||
return errno
|
||||
}
|
||||
|
||||
// Fdatasync on darwin platform invokes fcntl(F_FULLFSYNC) for actual persistence
|
||||
// on physical drive media.
|
||||
func Fdatasync(f *os.File) error {
|
||||
return Fsync(f)
|
||||
}
|
34
vendor/github.com/prometheus/tsdb/fileutil/sync_linux.go
generated
vendored
Normal file
34
vendor/github.com/prometheus/tsdb/fileutil/sync_linux.go
generated
vendored
Normal file
|
@ -0,0 +1,34 @@
|
|||
// Copyright 2016 The etcd Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build linux
|
||||
|
||||
package fileutil
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// Fsync is a wrapper around file.Sync(). Special handling is needed on darwin platform.
|
||||
func Fsync(f *os.File) error {
|
||||
return f.Sync()
|
||||
}
|
||||
|
||||
// Fdatasync is similar to fsync(), but does not flush modified metadata
|
||||
// unless that metadata is needed in order to allow a subsequent data retrieval
|
||||
// to be correctly handled.
|
||||
func Fdatasync(f *os.File) error {
|
||||
return syscall.Fdatasync(int(f.Fd()))
|
||||
}
|
28
vendor/github.com/prometheus/tsdb/go.mod
generated
vendored
Normal file
28
vendor/github.com/prometheus/tsdb/go.mod
generated
vendored
Normal file
|
@ -0,0 +1,28 @@
|
|||
module github.com/prometheus/tsdb
|
||||
|
||||
require (
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954
|
||||
github.com/go-kit/kit v0.8.0
|
||||
github.com/go-logfmt/logfmt v0.3.0 // indirect
|
||||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/gogo/protobuf v1.1.1 // indirect
|
||||
github.com/golang/protobuf v1.2.0 // indirect
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/oklog/ulid v1.3.1
|
||||
github.com/pkg/errors v0.8.0
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_golang v0.9.1
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910
|
||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce // indirect
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d // indirect
|
||||
github.com/stretchr/testify v1.2.2 // indirect
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
)
|
52
vendor/github.com/prometheus/tsdb/go.sum
generated
vendored
Normal file
52
vendor/github.com/prometheus/tsdb/go.sum
generated
vendored
Normal file
|
@ -0,0 +1,52 @@
|
|||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1 h1:K47Rk0v/fkEfwfQet2KWhscE0cJzjgCCDBG2KHZoVno=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce h1:X0jFYGnHemYDIW6jlc+fSI8f9Cg+jqCnClYP2WgZT/A=
|
||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d h1:GoAlyOgbOEIFdaDqxJVlbOQ1DtGmZWs/Qau0hIlk+WQ=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8 h1:YoY1wS6JYVRpIfFngRf2HHo9R9dAne3xbkGOQ5rJXjU=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
1719
vendor/github.com/prometheus/tsdb/head.go
generated
vendored
Normal file
1719
vendor/github.com/prometheus/tsdb/head.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load diff
1092
vendor/github.com/prometheus/tsdb/index/index.go
generated
vendored
Normal file
1092
vendor/github.com/prometheus/tsdb/index/index.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load diff
676
vendor/github.com/prometheus/tsdb/index/postings.go
generated
vendored
Normal file
676
vendor/github.com/prometheus/tsdb/index/postings.go
generated
vendored
Normal file
|
@ -0,0 +1,676 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package index
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"encoding/binary"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/prometheus/tsdb/labels"
|
||||
)
|
||||
|
||||
var allPostingsKey = labels.Label{}
|
||||
|
||||
// AllPostingsKey returns the label key that is used to store the postings list of all existing IDs.
|
||||
func AllPostingsKey() (name, value string) {
|
||||
return allPostingsKey.Name, allPostingsKey.Value
|
||||
}
|
||||
|
||||
// MemPostings holds postings list for series ID per label pair. They may be written
|
||||
// to out of order.
|
||||
// ensureOrder() must be called once before any reads are done. This allows for quick
|
||||
// unordered batch fills on startup.
|
||||
type MemPostings struct {
|
||||
mtx sync.RWMutex
|
||||
m map[string]map[string][]uint64
|
||||
ordered bool
|
||||
}
|
||||
|
||||
// NewMemPostings returns a memPostings that's ready for reads and writes.
|
||||
func NewMemPostings() *MemPostings {
|
||||
return &MemPostings{
|
||||
m: make(map[string]map[string][]uint64, 512),
|
||||
ordered: true,
|
||||
}
|
||||
}
|
||||
|
||||
// NewUnorderedMemPostings returns a memPostings that is not safe to be read from
|
||||
// until ensureOrder was called once.
|
||||
func NewUnorderedMemPostings() *MemPostings {
|
||||
return &MemPostings{
|
||||
m: make(map[string]map[string][]uint64, 512),
|
||||
ordered: false,
|
||||
}
|
||||
}
|
||||
|
||||
// SortedKeys returns a list of sorted label keys of the postings.
|
||||
func (p *MemPostings) SortedKeys() []labels.Label {
|
||||
p.mtx.RLock()
|
||||
keys := make([]labels.Label, 0, len(p.m))
|
||||
|
||||
for n, e := range p.m {
|
||||
for v := range e {
|
||||
keys = append(keys, labels.Label{Name: n, Value: v})
|
||||
}
|
||||
}
|
||||
p.mtx.RUnlock()
|
||||
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if d := strings.Compare(keys[i].Name, keys[j].Name); d != 0 {
|
||||
return d < 0
|
||||
}
|
||||
return keys[i].Value < keys[j].Value
|
||||
})
|
||||
return keys
|
||||
}
|
||||
|
||||
// Get returns a postings list for the given label pair.
|
||||
func (p *MemPostings) Get(name, value string) Postings {
|
||||
var lp []uint64
|
||||
p.mtx.RLock()
|
||||
l := p.m[name]
|
||||
if l != nil {
|
||||
lp = l[value]
|
||||
}
|
||||
p.mtx.RUnlock()
|
||||
|
||||
if lp == nil {
|
||||
return EmptyPostings()
|
||||
}
|
||||
return newListPostings(lp)
|
||||
}
|
||||
|
||||
// All returns a postings list over all documents ever added.
|
||||
func (p *MemPostings) All() Postings {
|
||||
return p.Get(AllPostingsKey())
|
||||
}
|
||||
|
||||
// EnsureOrder ensures that all postings lists are sorted. After it returns all further
|
||||
// calls to add and addFor will insert new IDs in a sorted manner.
|
||||
func (p *MemPostings) EnsureOrder() {
|
||||
p.mtx.Lock()
|
||||
defer p.mtx.Unlock()
|
||||
|
||||
if p.ordered {
|
||||
return
|
||||
}
|
||||
|
||||
n := runtime.GOMAXPROCS(0)
|
||||
workc := make(chan []uint64)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(n)
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
go func() {
|
||||
for l := range workc {
|
||||
sort.Slice(l, func(i, j int) bool { return l[i] < l[j] })
|
||||
}
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
|
||||
for _, e := range p.m {
|
||||
for _, l := range e {
|
||||
workc <- l
|
||||
}
|
||||
}
|
||||
close(workc)
|
||||
wg.Wait()
|
||||
|
||||
p.ordered = true
|
||||
}
|
||||
|
||||
// Delete removes all ids in the given map from the postings lists.
|
||||
func (p *MemPostings) Delete(deleted map[uint64]struct{}) {
|
||||
var keys, vals []string
|
||||
|
||||
// Collect all keys relevant for deletion once. New keys added afterwards
|
||||
// can by definition not be affected by any of the given deletes.
|
||||
p.mtx.RLock()
|
||||
for n := range p.m {
|
||||
keys = append(keys, n)
|
||||
}
|
||||
p.mtx.RUnlock()
|
||||
|
||||
for _, n := range keys {
|
||||
p.mtx.RLock()
|
||||
vals = vals[:0]
|
||||
for v := range p.m[n] {
|
||||
vals = append(vals, v)
|
||||
}
|
||||
p.mtx.RUnlock()
|
||||
|
||||
// For each posting we first analyse whether the postings list is affected by the deletes.
|
||||
// If yes, we actually reallocate a new postings list.
|
||||
for _, l := range vals {
|
||||
// Only lock for processing one postings list so we don't block reads for too long.
|
||||
p.mtx.Lock()
|
||||
|
||||
found := false
|
||||
for _, id := range p.m[n][l] {
|
||||
if _, ok := deleted[id]; ok {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
p.mtx.Unlock()
|
||||
continue
|
||||
}
|
||||
repl := make([]uint64, 0, len(p.m[n][l]))
|
||||
|
||||
for _, id := range p.m[n][l] {
|
||||
if _, ok := deleted[id]; !ok {
|
||||
repl = append(repl, id)
|
||||
}
|
||||
}
|
||||
if len(repl) > 0 {
|
||||
p.m[n][l] = repl
|
||||
} else {
|
||||
delete(p.m[n], l)
|
||||
}
|
||||
p.mtx.Unlock()
|
||||
}
|
||||
p.mtx.Lock()
|
||||
if len(p.m[n]) == 0 {
|
||||
delete(p.m, n)
|
||||
}
|
||||
p.mtx.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Iter calls f for each postings list. It aborts if f returns an error and returns it.
|
||||
func (p *MemPostings) Iter(f func(labels.Label, Postings) error) error {
|
||||
p.mtx.RLock()
|
||||
defer p.mtx.RUnlock()
|
||||
|
||||
for n, e := range p.m {
|
||||
for v, p := range e {
|
||||
if err := f(labels.Label{Name: n, Value: v}, newListPostings(p)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add a label set to the postings index.
|
||||
func (p *MemPostings) Add(id uint64, lset labels.Labels) {
|
||||
p.mtx.Lock()
|
||||
|
||||
for _, l := range lset {
|
||||
p.addFor(id, l)
|
||||
}
|
||||
p.addFor(id, allPostingsKey)
|
||||
|
||||
p.mtx.Unlock()
|
||||
}
|
||||
|
||||
func (p *MemPostings) addFor(id uint64, l labels.Label) {
|
||||
nm, ok := p.m[l.Name]
|
||||
if !ok {
|
||||
nm = map[string][]uint64{}
|
||||
p.m[l.Name] = nm
|
||||
}
|
||||
list := append(nm[l.Value], id)
|
||||
nm[l.Value] = list
|
||||
|
||||
if !p.ordered {
|
||||
return
|
||||
}
|
||||
// There is no guarantee that no higher ID was inserted before as they may
|
||||
// be generated independently before adding them to postings.
|
||||
// We repair order violations on insert. The invariant is that the first n-1
|
||||
// items in the list are already sorted.
|
||||
for i := len(list) - 1; i >= 1; i-- {
|
||||
if list[i] >= list[i-1] {
|
||||
break
|
||||
}
|
||||
list[i], list[i-1] = list[i-1], list[i]
|
||||
}
|
||||
}
|
||||
|
||||
// ExpandPostings returns the postings expanded as a slice.
|
||||
func ExpandPostings(p Postings) (res []uint64, err error) {
|
||||
for p.Next() {
|
||||
res = append(res, p.At())
|
||||
}
|
||||
return res, p.Err()
|
||||
}
|
||||
|
||||
// Postings provides iterative access over a postings list.
|
||||
type Postings interface {
|
||||
// Next advances the iterator and returns true if another value was found.
|
||||
Next() bool
|
||||
|
||||
// Seek advances the iterator to value v or greater and returns
|
||||
// true if a value was found.
|
||||
Seek(v uint64) bool
|
||||
|
||||
// At returns the value at the current iterator position.
|
||||
At() uint64
|
||||
|
||||
// Err returns the last error of the iterator.
|
||||
Err() error
|
||||
}
|
||||
|
||||
// errPostings is an empty iterator that always errors.
|
||||
type errPostings struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e errPostings) Next() bool { return false }
|
||||
func (e errPostings) Seek(uint64) bool { return false }
|
||||
func (e errPostings) At() uint64 { return 0 }
|
||||
func (e errPostings) Err() error { return e.err }
|
||||
|
||||
var emptyPostings = errPostings{}
|
||||
|
||||
// EmptyPostings returns a postings list that's always empty.
|
||||
func EmptyPostings() Postings {
|
||||
return emptyPostings
|
||||
}
|
||||
|
||||
// ErrPostings returns new postings that immediately error.
|
||||
func ErrPostings(err error) Postings {
|
||||
return errPostings{err}
|
||||
}
|
||||
|
||||
// Intersect returns a new postings list over the intersection of the
|
||||
// input postings.
|
||||
func Intersect(its ...Postings) Postings {
|
||||
if len(its) == 0 {
|
||||
return emptyPostings
|
||||
}
|
||||
if len(its) == 1 {
|
||||
return its[0]
|
||||
}
|
||||
l := len(its) / 2
|
||||
return newIntersectPostings(Intersect(its[:l]...), Intersect(its[l:]...))
|
||||
}
|
||||
|
||||
type intersectPostings struct {
|
||||
a, b Postings
|
||||
cur uint64
|
||||
}
|
||||
|
||||
func newIntersectPostings(a, b Postings) *intersectPostings {
|
||||
return &intersectPostings{a: a, b: b}
|
||||
}
|
||||
|
||||
func (it *intersectPostings) At() uint64 {
|
||||
return it.cur
|
||||
}
|
||||
|
||||
func (it *intersectPostings) doNext(id uint64) bool {
|
||||
for {
|
||||
if !it.b.Seek(id) {
|
||||
return false
|
||||
}
|
||||
if vb := it.b.At(); vb != id {
|
||||
if !it.a.Seek(vb) {
|
||||
return false
|
||||
}
|
||||
id = it.a.At()
|
||||
if vb != id {
|
||||
continue
|
||||
}
|
||||
}
|
||||
it.cur = id
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func (it *intersectPostings) Next() bool {
|
||||
if !it.a.Next() {
|
||||
return false
|
||||
}
|
||||
return it.doNext(it.a.At())
|
||||
}
|
||||
|
||||
func (it *intersectPostings) Seek(id uint64) bool {
|
||||
if !it.a.Seek(id) {
|
||||
return false
|
||||
}
|
||||
return it.doNext(it.a.At())
|
||||
}
|
||||
|
||||
func (it *intersectPostings) Err() error {
|
||||
if it.a.Err() != nil {
|
||||
return it.a.Err()
|
||||
}
|
||||
return it.b.Err()
|
||||
}
|
||||
|
||||
// Merge returns a new iterator over the union of the input iterators.
|
||||
func Merge(its ...Postings) Postings {
|
||||
if len(its) == 0 {
|
||||
return EmptyPostings()
|
||||
}
|
||||
if len(its) == 1 {
|
||||
return its[0]
|
||||
}
|
||||
return newMergedPostings(its)
|
||||
}
|
||||
|
||||
type postingsHeap []Postings
|
||||
|
||||
func (h postingsHeap) Len() int { return len(h) }
|
||||
func (h postingsHeap) Less(i, j int) bool { return h[i].At() < h[j].At() }
|
||||
func (h *postingsHeap) Swap(i, j int) { (*h)[i], (*h)[j] = (*h)[j], (*h)[i] }
|
||||
|
||||
func (h *postingsHeap) Push(x interface{}) {
|
||||
*h = append(*h, x.(Postings))
|
||||
}
|
||||
|
||||
func (h *postingsHeap) Pop() interface{} {
|
||||
old := *h
|
||||
n := len(old)
|
||||
x := old[n-1]
|
||||
*h = old[0 : n-1]
|
||||
return x
|
||||
}
|
||||
|
||||
type mergedPostings struct {
|
||||
h postingsHeap
|
||||
initilized bool
|
||||
heaped bool
|
||||
cur uint64
|
||||
err error
|
||||
}
|
||||
|
||||
func newMergedPostings(p []Postings) *mergedPostings {
|
||||
ph := make(postingsHeap, 0, len(p))
|
||||
for _, it := range p {
|
||||
if it.Next() {
|
||||
ph = append(ph, it)
|
||||
} else {
|
||||
if it.Err() != nil {
|
||||
return &mergedPostings{err: it.Err()}
|
||||
}
|
||||
}
|
||||
}
|
||||
return &mergedPostings{h: ph}
|
||||
}
|
||||
|
||||
func (it *mergedPostings) Next() bool {
|
||||
if it.h.Len() == 0 || it.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if !it.heaped {
|
||||
heap.Init(&it.h)
|
||||
it.heaped = true
|
||||
}
|
||||
// The user must issue an initial Next.
|
||||
if !it.initilized {
|
||||
it.cur = it.h[0].At()
|
||||
it.initilized = true
|
||||
return true
|
||||
}
|
||||
|
||||
for {
|
||||
cur := it.h[0]
|
||||
if !cur.Next() {
|
||||
heap.Pop(&it.h)
|
||||
if cur.Err() != nil {
|
||||
it.err = cur.Err()
|
||||
return false
|
||||
}
|
||||
if it.h.Len() == 0 {
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
// Value of top of heap has changed, re-heapify.
|
||||
heap.Fix(&it.h, 0)
|
||||
}
|
||||
|
||||
if it.h[0].At() != it.cur {
|
||||
it.cur = it.h[0].At()
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (it *mergedPostings) Seek(id uint64) bool {
|
||||
if it.h.Len() == 0 || it.err != nil {
|
||||
return false
|
||||
}
|
||||
if !it.initilized {
|
||||
if !it.Next() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if it.cur >= id {
|
||||
return true
|
||||
}
|
||||
// Heapifying when there is lots of Seeks is inefficient,
|
||||
// mark to be re-heapified on the Next() call.
|
||||
it.heaped = false
|
||||
newH := make(postingsHeap, 0, len(it.h))
|
||||
lowest := ^uint64(0)
|
||||
for _, i := range it.h {
|
||||
if i.Seek(id) {
|
||||
newH = append(newH, i)
|
||||
if i.At() < lowest {
|
||||
lowest = i.At()
|
||||
}
|
||||
} else {
|
||||
if i.Err() != nil {
|
||||
it.err = i.Err()
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
it.h = newH
|
||||
if len(it.h) == 0 {
|
||||
return false
|
||||
}
|
||||
it.cur = lowest
|
||||
return true
|
||||
}
|
||||
|
||||
func (it mergedPostings) At() uint64 {
|
||||
return it.cur
|
||||
}
|
||||
|
||||
func (it mergedPostings) Err() error {
|
||||
return it.err
|
||||
}
|
||||
|
||||
// Without returns a new postings list that contains all elements from the full list that
|
||||
// are not in the drop list
|
||||
func Without(full, drop Postings) Postings {
|
||||
return newRemovedPostings(full, drop)
|
||||
}
|
||||
|
||||
type removedPostings struct {
|
||||
full, remove Postings
|
||||
|
||||
cur uint64
|
||||
|
||||
initialized bool
|
||||
fok, rok bool
|
||||
}
|
||||
|
||||
func newRemovedPostings(full, remove Postings) *removedPostings {
|
||||
return &removedPostings{
|
||||
full: full,
|
||||
remove: remove,
|
||||
}
|
||||
}
|
||||
|
||||
func (rp *removedPostings) At() uint64 {
|
||||
return rp.cur
|
||||
}
|
||||
|
||||
func (rp *removedPostings) Next() bool {
|
||||
if !rp.initialized {
|
||||
rp.fok = rp.full.Next()
|
||||
rp.rok = rp.remove.Next()
|
||||
rp.initialized = true
|
||||
}
|
||||
for {
|
||||
if !rp.fok {
|
||||
return false
|
||||
}
|
||||
|
||||
if !rp.rok {
|
||||
rp.cur = rp.full.At()
|
||||
rp.fok = rp.full.Next()
|
||||
return true
|
||||
}
|
||||
|
||||
fcur, rcur := rp.full.At(), rp.remove.At()
|
||||
if fcur < rcur {
|
||||
rp.cur = fcur
|
||||
rp.fok = rp.full.Next()
|
||||
|
||||
return true
|
||||
} else if rcur < fcur {
|
||||
// Forward the remove postings to the right position.
|
||||
rp.rok = rp.remove.Seek(fcur)
|
||||
} else {
|
||||
// Skip the current posting.
|
||||
rp.fok = rp.full.Next()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (rp *removedPostings) Seek(id uint64) bool {
|
||||
if rp.cur >= id {
|
||||
return true
|
||||
}
|
||||
|
||||
rp.fok = rp.full.Seek(id)
|
||||
rp.rok = rp.remove.Seek(id)
|
||||
rp.initialized = true
|
||||
|
||||
return rp.Next()
|
||||
}
|
||||
|
||||
func (rp *removedPostings) Err() error {
|
||||
if rp.full.Err() != nil {
|
||||
return rp.full.Err()
|
||||
}
|
||||
|
||||
return rp.remove.Err()
|
||||
}
|
||||
|
||||
// listPostings implements the Postings interface over a plain list.
|
||||
type listPostings struct {
|
||||
list []uint64
|
||||
cur uint64
|
||||
}
|
||||
|
||||
func NewListPostings(list []uint64) Postings {
|
||||
return newListPostings(list)
|
||||
}
|
||||
|
||||
func newListPostings(list []uint64) *listPostings {
|
||||
return &listPostings{list: list}
|
||||
}
|
||||
|
||||
func (it *listPostings) At() uint64 {
|
||||
return it.cur
|
||||
}
|
||||
|
||||
func (it *listPostings) Next() bool {
|
||||
if len(it.list) > 0 {
|
||||
it.cur = it.list[0]
|
||||
it.list = it.list[1:]
|
||||
return true
|
||||
}
|
||||
it.cur = 0
|
||||
return false
|
||||
}
|
||||
|
||||
func (it *listPostings) Seek(x uint64) bool {
|
||||
// If the current value satisfies, then return.
|
||||
if it.cur >= x {
|
||||
return true
|
||||
}
|
||||
if len(it.list) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Do binary search between current position and end.
|
||||
i := sort.Search(len(it.list), func(i int) bool {
|
||||
return it.list[i] >= x
|
||||
})
|
||||
if i < len(it.list) {
|
||||
it.cur = it.list[i]
|
||||
it.list = it.list[i+1:]
|
||||
return true
|
||||
}
|
||||
it.list = nil
|
||||
return false
|
||||
}
|
||||
|
||||
func (it *listPostings) Err() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// bigEndianPostings implements the Postings interface over a byte stream of
|
||||
// big endian numbers.
|
||||
type bigEndianPostings struct {
|
||||
list []byte
|
||||
cur uint32
|
||||
}
|
||||
|
||||
func newBigEndianPostings(list []byte) *bigEndianPostings {
|
||||
return &bigEndianPostings{list: list}
|
||||
}
|
||||
|
||||
func (it *bigEndianPostings) At() uint64 {
|
||||
return uint64(it.cur)
|
||||
}
|
||||
|
||||
func (it *bigEndianPostings) Next() bool {
|
||||
if len(it.list) >= 4 {
|
||||
it.cur = binary.BigEndian.Uint32(it.list)
|
||||
it.list = it.list[4:]
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (it *bigEndianPostings) Seek(x uint64) bool {
|
||||
if uint64(it.cur) >= x {
|
||||
return true
|
||||
}
|
||||
|
||||
num := len(it.list) / 4
|
||||
// Do binary search between current position and end.
|
||||
i := sort.Search(num, func(i int) bool {
|
||||
return binary.BigEndian.Uint32(it.list[i*4:]) >= uint32(x)
|
||||
})
|
||||
if i < num {
|
||||
j := i * 4
|
||||
it.cur = binary.BigEndian.Uint32(it.list[j:])
|
||||
it.list = it.list[j+4:]
|
||||
return true
|
||||
}
|
||||
it.list = nil
|
||||
return false
|
||||
}
|
||||
|
||||
func (it *bigEndianPostings) Err() error {
|
||||
return nil
|
||||
}
|
214
vendor/github.com/prometheus/tsdb/labels/labels.go
generated
vendored
Normal file
214
vendor/github.com/prometheus/tsdb/labels/labels.go
generated
vendored
Normal file
|
@ -0,0 +1,214 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package labels
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/cespare/xxhash"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const sep = '\xff'
|
||||
|
||||
// Label is a key/value pair of strings.
|
||||
type Label struct {
|
||||
Name, Value string
|
||||
}
|
||||
|
||||
// Labels is a sorted set of labels. Order has to be guaranteed upon
|
||||
// instantiation.
|
||||
type Labels []Label
|
||||
|
||||
func (ls Labels) Len() int { return len(ls) }
|
||||
func (ls Labels) Swap(i, j int) { ls[i], ls[j] = ls[j], ls[i] }
|
||||
func (ls Labels) Less(i, j int) bool { return ls[i].Name < ls[j].Name }
|
||||
|
||||
func (ls Labels) String() string {
|
||||
var b bytes.Buffer
|
||||
|
||||
b.WriteByte('{')
|
||||
for i, l := range ls {
|
||||
if i > 0 {
|
||||
b.WriteByte(',')
|
||||
}
|
||||
b.WriteString(l.Name)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(strconv.Quote(l.Value))
|
||||
}
|
||||
b.WriteByte('}')
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Hash returns a hash value for the label set.
|
||||
func (ls Labels) Hash() uint64 {
|
||||
b := make([]byte, 0, 1024)
|
||||
|
||||
for _, v := range ls {
|
||||
b = append(b, v.Name...)
|
||||
b = append(b, sep)
|
||||
b = append(b, v.Value...)
|
||||
b = append(b, sep)
|
||||
}
|
||||
return xxhash.Sum64(b)
|
||||
}
|
||||
|
||||
// Get returns the value for the label with the given name.
|
||||
// Returns an empty string if the label doesn't exist.
|
||||
func (ls Labels) Get(name string) string {
|
||||
for _, l := range ls {
|
||||
if l.Name == name {
|
||||
return l.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Equals returns whether the two label sets are equal.
|
||||
func (ls Labels) Equals(o Labels) bool {
|
||||
if len(ls) != len(o) {
|
||||
return false
|
||||
}
|
||||
for i, l := range ls {
|
||||
if o[i] != l {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Map returns a string map of the labels.
|
||||
func (ls Labels) Map() map[string]string {
|
||||
m := make(map[string]string, len(ls))
|
||||
for _, l := range ls {
|
||||
m[l.Name] = l.Value
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// New returns a sorted Labels from the given labels.
|
||||
// The caller has to guarantee that all label names are unique.
|
||||
func New(ls ...Label) Labels {
|
||||
set := make(Labels, 0, len(ls))
|
||||
for _, l := range ls {
|
||||
set = append(set, l)
|
||||
}
|
||||
sort.Sort(set)
|
||||
|
||||
return set
|
||||
}
|
||||
|
||||
// FromMap returns new sorted Labels from the given map.
|
||||
func FromMap(m map[string]string) Labels {
|
||||
l := make(Labels, 0, len(m))
|
||||
for k, v := range m {
|
||||
l = append(l, Label{Name: k, Value: v})
|
||||
}
|
||||
sort.Sort(l)
|
||||
|
||||
return l
|
||||
}
|
||||
|
||||
// FromStrings creates new labels from pairs of strings.
|
||||
func FromStrings(ss ...string) Labels {
|
||||
if len(ss)%2 != 0 {
|
||||
panic("invalid number of strings")
|
||||
}
|
||||
var res Labels
|
||||
for i := 0; i < len(ss); i += 2 {
|
||||
res = append(res, Label{Name: ss[i], Value: ss[i+1]})
|
||||
}
|
||||
|
||||
sort.Sort(res)
|
||||
return res
|
||||
}
|
||||
|
||||
// Compare compares the two label sets.
|
||||
// The result will be 0 if a==b, <0 if a < b, and >0 if a > b.
|
||||
func Compare(a, b Labels) int {
|
||||
l := len(a)
|
||||
if len(b) < l {
|
||||
l = len(b)
|
||||
}
|
||||
|
||||
for i := 0; i < l; i++ {
|
||||
if d := strings.Compare(a[i].Name, b[i].Name); d != 0 {
|
||||
return d
|
||||
}
|
||||
if d := strings.Compare(a[i].Value, b[i].Value); d != 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
// If all labels so far were in common, the set with fewer labels comes first.
|
||||
return len(a) - len(b)
|
||||
}
|
||||
|
||||
// Slice is a sortable slice of label sets.
|
||||
type Slice []Labels
|
||||
|
||||
func (s Slice) Len() int { return len(s) }
|
||||
func (s Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
func (s Slice) Less(i, j int) bool { return Compare(s[i], s[j]) < 0 }
|
||||
|
||||
// ReadLabels reads up to n label sets in a JSON formatted file fn. It is mostly useful
|
||||
// to load testing data.
|
||||
func ReadLabels(fn string, n int) ([]Labels, error) {
|
||||
f, err := os.Open(fn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
|
||||
var mets []Labels
|
||||
hashes := map[uint64]struct{}{}
|
||||
i := 0
|
||||
|
||||
for scanner.Scan() && i < n {
|
||||
m := make(Labels, 0, 10)
|
||||
|
||||
r := strings.NewReplacer("\"", "", "{", "", "}", "")
|
||||
s := r.Replace(scanner.Text())
|
||||
|
||||
labelChunks := strings.Split(s, ",")
|
||||
for _, labelChunk := range labelChunks {
|
||||
split := strings.Split(labelChunk, ":")
|
||||
m = append(m, Label{Name: split[0], Value: split[1]})
|
||||
}
|
||||
// Order of the k/v labels matters, don't assume we'll always receive them already sorted.
|
||||
sort.Sort(m)
|
||||
|
||||
h := m.Hash()
|
||||
if _, ok := hashes[h]; ok {
|
||||
continue
|
||||
}
|
||||
mets = append(mets, m)
|
||||
hashes[h] = struct{}{}
|
||||
i++
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if i != n {
|
||||
return mets, errors.Errorf("requested %d metrics but found %d", n, i)
|
||||
}
|
||||
return mets, nil
|
||||
}
|
100
vendor/github.com/prometheus/tsdb/labels/selector.go
generated
vendored
Normal file
100
vendor/github.com/prometheus/tsdb/labels/selector.go
generated
vendored
Normal file
|
@ -0,0 +1,100 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package labels
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// Selector holds constraints for matching against a label set.
|
||||
type Selector []Matcher
|
||||
|
||||
// Matches returns whether the labels satisfy all matchers.
|
||||
func (s Selector) Matches(labels Labels) bool {
|
||||
for _, m := range s {
|
||||
if v := labels.Get(m.Name()); !m.Matches(v) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Matcher specifies a constraint for the value of a label.
|
||||
type Matcher interface {
|
||||
// Name returns the label name the matcher should apply to.
|
||||
Name() string
|
||||
// Matches checks whether a value fulfills the constraints.
|
||||
Matches(v string) bool
|
||||
}
|
||||
|
||||
// EqualMatcher matches on equality.
|
||||
type EqualMatcher struct {
|
||||
name, value string
|
||||
}
|
||||
|
||||
// Name implements Matcher interface.
|
||||
func (m *EqualMatcher) Name() string { return m.name }
|
||||
|
||||
// Matches implements Matcher interface.
|
||||
func (m *EqualMatcher) Matches(v string) bool { return v == m.value }
|
||||
|
||||
// Value returns the matched value.
|
||||
func (m *EqualMatcher) Value() string { return m.value }
|
||||
|
||||
// NewEqualMatcher returns a new matcher matching an exact label value.
|
||||
func NewEqualMatcher(name, value string) Matcher {
|
||||
return &EqualMatcher{name: name, value: value}
|
||||
}
|
||||
|
||||
type regexpMatcher struct {
|
||||
name string
|
||||
re *regexp.Regexp
|
||||
}
|
||||
|
||||
func (m *regexpMatcher) Name() string { return m.name }
|
||||
func (m *regexpMatcher) Matches(v string) bool { return m.re.MatchString(v) }
|
||||
|
||||
// NewRegexpMatcher returns a new matcher verifying that a value matches
|
||||
// the regular expression pattern.
|
||||
func NewRegexpMatcher(name, pattern string) (Matcher, error) {
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ®expMatcher{name: name, re: re}, nil
|
||||
}
|
||||
|
||||
// NewRegexpMatcher returns a new matcher verifying that a value matches
|
||||
// the regular expression pattern. Will panic if the pattern is not a valid
|
||||
// regular expression.
|
||||
func NewMustRegexpMatcher(name, pattern string) Matcher {
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return ®expMatcher{name: name, re: re}
|
||||
|
||||
}
|
||||
|
||||
// notMatcher inverts the matching result for a matcher.
|
||||
type notMatcher struct {
|
||||
Matcher
|
||||
}
|
||||
|
||||
func (m *notMatcher) Matches(v string) bool { return !m.Matcher.Matches(v) }
|
||||
|
||||
// Not inverts the matcher's matching result.
|
||||
func Not(m Matcher) Matcher {
|
||||
return ¬Matcher{m}
|
||||
}
|
1091
vendor/github.com/prometheus/tsdb/querier.go
generated
vendored
Normal file
1091
vendor/github.com/prometheus/tsdb/querier.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load diff
208
vendor/github.com/prometheus/tsdb/record.go
generated
vendored
Normal file
208
vendor/github.com/prometheus/tsdb/record.go
generated
vendored
Normal file
|
@ -0,0 +1,208 @@
|
|||
// Copyright 2018 The Prometheus Authors
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tsdb
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/tsdb/encoding"
|
||||
"github.com/prometheus/tsdb/labels"
|
||||
)
|
||||
|
||||
// RecordType represents the data type of a record.
|
||||
type RecordType uint8
|
||||
|
||||
const (
|
||||
// RecordInvalid is returned for unrecognised WAL record types.
|
||||
RecordInvalid RecordType = 255
|
||||
// RecordSeries is used to match WAL records of type Series.
|
||||
RecordSeries RecordType = 1
|
||||
// RecordSamples is used to match WAL records of type Samples.
|
||||
RecordSamples RecordType = 2
|
||||
// RecordTombstones is used to match WAL records of type Tombstones.
|
||||
RecordTombstones RecordType = 3
|
||||
)
|
||||
|
||||
// RecordDecoder decodes series, sample, and tombstone records.
|
||||
// The zero value is ready to use.
|
||||
type RecordDecoder struct {
|
||||
}
|
||||
|
||||
// Type returns the type of the record.
|
||||
// Return RecordInvalid if no valid record type is found.
|
||||
func (d *RecordDecoder) Type(rec []byte) RecordType {
|
||||
if len(rec) < 1 {
|
||||
return RecordInvalid
|
||||
}
|
||||
switch t := RecordType(rec[0]); t {
|
||||
case RecordSeries, RecordSamples, RecordTombstones:
|
||||
return t
|
||||
}
|
||||
return RecordInvalid
|
||||
}
|
||||
|
||||
// Series appends series in rec to the given slice.
|
||||
func (d *RecordDecoder) Series(rec []byte, series []RefSeries) ([]RefSeries, error) {
|
||||
dec := encoding.Decbuf{B: rec}
|
||||
|
||||
if RecordType(dec.Byte()) != RecordSeries {
|
||||
return nil, errors.New("invalid record type")
|
||||
}
|
||||
for len(dec.B) > 0 && dec.Err() == nil {
|
||||
ref := dec.Be64()
|
||||
|
||||
lset := make(labels.Labels, dec.Uvarint())
|
||||
|
||||
for i := range lset {
|
||||
lset[i].Name = dec.UvarintStr()
|
||||
lset[i].Value = dec.UvarintStr()
|
||||
}
|
||||
sort.Sort(lset)
|
||||
|
||||
series = append(series, RefSeries{
|
||||
Ref: ref,
|
||||
Labels: lset,
|
||||
})
|
||||
}
|
||||
if dec.Err() != nil {
|
||||
return nil, dec.Err()
|
||||
}
|
||||
if len(dec.B) > 0 {
|
||||
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
||||
}
|
||||
return series, nil
|
||||
}
|
||||
|
||||
// Samples appends samples in rec to the given slice.
|
||||
func (d *RecordDecoder) Samples(rec []byte, samples []RefSample) ([]RefSample, error) {
|
||||
dec := encoding.Decbuf{B: rec}
|
||||
|
||||
if RecordType(dec.Byte()) != RecordSamples {
|
||||
return nil, errors.New("invalid record type")
|
||||
}
|
||||
if dec.Len() == 0 {
|
||||
return samples, nil
|
||||
}
|
||||
var (
|
||||
baseRef = dec.Be64()
|
||||
baseTime = dec.Be64int64()
|
||||
)
|
||||
for len(dec.B) > 0 && dec.Err() == nil {
|
||||
dref := dec.Varint64()
|
||||
dtime := dec.Varint64()
|
||||
val := dec.Be64()
|
||||
|
||||
samples = append(samples, RefSample{
|
||||
Ref: uint64(int64(baseRef) + dref),
|
||||
T: baseTime + dtime,
|
||||
V: math.Float64frombits(val),
|
||||
})
|
||||
}
|
||||
|
||||
if dec.Err() != nil {
|
||||
return nil, errors.Wrapf(dec.Err(), "decode error after %d samples", len(samples))
|
||||
}
|
||||
if len(dec.B) > 0 {
|
||||
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
||||
}
|
||||
return samples, nil
|
||||
}
|
||||
|
||||
// Tombstones appends tombstones in rec to the given slice.
|
||||
func (d *RecordDecoder) Tombstones(rec []byte, tstones []Stone) ([]Stone, error) {
|
||||
dec := encoding.Decbuf{B: rec}
|
||||
|
||||
if RecordType(dec.Byte()) != RecordTombstones {
|
||||
return nil, errors.New("invalid record type")
|
||||
}
|
||||
for dec.Len() > 0 && dec.Err() == nil {
|
||||
tstones = append(tstones, Stone{
|
||||
ref: dec.Be64(),
|
||||
intervals: Intervals{
|
||||
{Mint: dec.Varint64(), Maxt: dec.Varint64()},
|
||||
},
|
||||
})
|
||||
}
|
||||
if dec.Err() != nil {
|
||||
return nil, dec.Err()
|
||||
}
|
||||
if len(dec.B) > 0 {
|
||||
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
||||
}
|
||||
return tstones, nil
|
||||
}
|
||||
|
||||
// RecordEncoder encodes series, sample, and tombstones records.
|
||||
// The zero value is ready to use.
|
||||
type RecordEncoder struct {
|
||||
}
|
||||
|
||||
// Series appends the encoded series to b and returns the resulting slice.
|
||||
func (e *RecordEncoder) Series(series []RefSeries, b []byte) []byte {
|
||||
buf := encoding.Encbuf{B: b}
|
||||
buf.PutByte(byte(RecordSeries))
|
||||
|
||||
for _, s := range series {
|
||||
buf.PutBE64(s.Ref)
|
||||
buf.PutUvarint(len(s.Labels))
|
||||
|
||||
for _, l := range s.Labels {
|
||||
buf.PutUvarintStr(l.Name)
|
||||
buf.PutUvarintStr(l.Value)
|
||||
}
|
||||
}
|
||||
return buf.Get()
|
||||
}
|
||||
|
||||
// Samples appends the encoded samples to b and returns the resulting slice.
|
||||
func (e *RecordEncoder) Samples(samples []RefSample, b []byte) []byte {
|
||||
buf := encoding.Encbuf{B: b}
|
||||
buf.PutByte(byte(RecordSamples))
|
||||
|
||||
if len(samples) == 0 {
|
||||
return buf.Get()
|
||||
}
|
||||
|
||||
// Store base timestamp and base reference number of first sample.
|
||||
// All samples encode their timestamp and ref as delta to those.
|
||||
first := samples[0]
|
||||
|
||||
buf.PutBE64(first.Ref)
|
||||
buf.PutBE64int64(first.T)
|
||||
|
||||
for _, s := range samples {
|
||||
buf.PutVarint64(int64(s.Ref) - int64(first.Ref))
|
||||
buf.PutVarint64(s.T - first.T)
|
||||
buf.PutBE64(math.Float64bits(s.V))
|
||||
}
|
||||
return buf.Get()
|
||||
}
|
||||
|
||||
// Tombstones appends the encoded tombstones to b and returns the resulting slice.
|
||||
func (e *RecordEncoder) Tombstones(tstones []Stone, b []byte) []byte {
|
||||
buf := encoding.Encbuf{B: b}
|
||||
buf.PutByte(byte(RecordTombstones))
|
||||
|
||||
for _, s := range tstones {
|
||||
for _, iv := range s.intervals {
|
||||
buf.PutBE64(s.ref)
|
||||
buf.PutVarint64(iv.Mint)
|
||||
buf.PutVarint64(iv.Maxt)
|
||||
}
|
||||
}
|
||||
return buf.Get()
|
||||
}
|
113
vendor/github.com/prometheus/tsdb/repair.go
generated
vendored
Normal file
113
vendor/github.com/prometheus/tsdb/repair.go
generated
vendored
Normal file
|
@ -0,0 +1,113 @@
|
|||
// Copyright 2018 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tsdb
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/go-kit/kit/log"
|
||||
"github.com/go-kit/kit/log/level"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/tsdb/fileutil"
|
||||
)
|
||||
|
||||
// repairBadIndexVersion repairs an issue in index and meta.json persistence introduced in
|
||||
// commit 129773b41a565fde5156301e37f9a87158030443.
|
||||
func repairBadIndexVersion(logger log.Logger, dir string) error {
|
||||
// All blocks written by Prometheus 2.1 with a meta.json version of 2 are affected.
|
||||
// We must actually set the index file version to 2 and revert the meta.json version back to 1.
|
||||
dirs, err := blockDirs(dir)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "list block dirs in %q", dir)
|
||||
}
|
||||
|
||||
wrapErr := func(err error, d string) error {
|
||||
return errors.Wrapf(err, "block dir: %q", d)
|
||||
}
|
||||
for _, d := range dirs {
|
||||
meta, err := readBogusMetaFile(d)
|
||||
if err != nil {
|
||||
return wrapErr(err, d)
|
||||
}
|
||||
if meta.Version == 1 {
|
||||
level.Info(logger).Log(
|
||||
"msg", "found healthy block",
|
||||
"mint", meta.MinTime,
|
||||
"maxt", meta.MaxTime,
|
||||
"ulid", meta.ULID,
|
||||
)
|
||||
continue
|
||||
}
|
||||
level.Info(logger).Log(
|
||||
"msg", "fixing broken block",
|
||||
"mint", meta.MinTime,
|
||||
"maxt", meta.MaxTime,
|
||||
"ulid", meta.ULID,
|
||||
)
|
||||
|
||||
repl, err := os.Create(filepath.Join(d, "index.repaired"))
|
||||
if err != nil {
|
||||
return wrapErr(err, d)
|
||||
}
|
||||
broken, err := os.Open(filepath.Join(d, indexFilename))
|
||||
if err != nil {
|
||||
return wrapErr(err, d)
|
||||
}
|
||||
if _, err := io.Copy(repl, broken); err != nil {
|
||||
return wrapErr(err, d)
|
||||
}
|
||||
// Set the 5th byte to 2 to indicate the correct file format version.
|
||||
if _, err := repl.WriteAt([]byte{2}, 4); err != nil {
|
||||
return wrapErr(err, d)
|
||||
}
|
||||
if err := fileutil.Fsync(repl); err != nil {
|
||||
return wrapErr(err, d)
|
||||
}
|
||||
if err := repl.Close(); err != nil {
|
||||
return wrapErr(err, d)
|
||||
}
|
||||
if err := broken.Close(); err != nil {
|
||||
return wrapErr(err, d)
|
||||
}
|
||||
if err := renameFile(repl.Name(), broken.Name()); err != nil {
|
||||
return wrapErr(err, d)
|
||||
}
|
||||
// Reset version of meta.json to 1.
|
||||
meta.Version = 1
|
||||
if err := writeMetaFile(d, meta); err != nil {
|
||||
return wrapErr(err, d)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readBogusMetaFile(dir string) (*BlockMeta, error) {
|
||||
b, err := ioutil.ReadFile(filepath.Join(dir, metaFilename))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m BlockMeta
|
||||
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m.Version != 1 && m.Version != 2 {
|
||||
return nil, errors.Errorf("unexpected meta file version %d", m.Version)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
299
vendor/github.com/prometheus/tsdb/tombstones.go
generated
vendored
Normal file
299
vendor/github.com/prometheus/tsdb/tombstones.go
generated
vendored
Normal file
|
@ -0,0 +1,299 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tsdb
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/tsdb/encoding"
|
||||
)
|
||||
|
||||
const tombstoneFilename = "tombstones"
|
||||
|
||||
const (
|
||||
// MagicTombstone is 4 bytes at the head of a tombstone file.
|
||||
MagicTombstone = 0x0130BA30
|
||||
|
||||
tombstoneFormatV1 = 1
|
||||
)
|
||||
|
||||
// TombstoneReader gives access to tombstone intervals by series reference.
|
||||
type TombstoneReader interface {
|
||||
// Get returns deletion intervals for the series with the given reference.
|
||||
Get(ref uint64) (Intervals, error)
|
||||
|
||||
// Iter calls the given function for each encountered interval.
|
||||
Iter(func(uint64, Intervals) error) error
|
||||
|
||||
// Total returns the total count of tombstones.
|
||||
Total() uint64
|
||||
|
||||
// Close any underlying resources
|
||||
Close() error
|
||||
}
|
||||
|
||||
func writeTombstoneFile(dir string, tr TombstoneReader) error {
|
||||
path := filepath.Join(dir, tombstoneFilename)
|
||||
tmp := path + ".tmp"
|
||||
hash := newCRC32()
|
||||
|
||||
f, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if f != nil {
|
||||
f.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
buf := encoding.Encbuf{B: make([]byte, 3*binary.MaxVarintLen64)}
|
||||
buf.Reset()
|
||||
// Write the meta.
|
||||
buf.PutBE32(MagicTombstone)
|
||||
buf.PutByte(tombstoneFormatV1)
|
||||
_, err = f.Write(buf.Get())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mw := io.MultiWriter(f, hash)
|
||||
|
||||
if err := tr.Iter(func(ref uint64, ivs Intervals) error {
|
||||
for _, iv := range ivs {
|
||||
buf.Reset()
|
||||
|
||||
buf.PutUvarint64(ref)
|
||||
buf.PutVarint64(iv.Mint)
|
||||
buf.PutVarint64(iv.Maxt)
|
||||
|
||||
_, err = mw.Write(buf.Get())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("error writing tombstones: %v", err)
|
||||
}
|
||||
|
||||
_, err = f.Write(hash.Sum(nil))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
f = nil
|
||||
return renameFile(tmp, path)
|
||||
}
|
||||
|
||||
// Stone holds the information on the posting and time-range
|
||||
// that is deleted.
|
||||
type Stone struct {
|
||||
ref uint64
|
||||
intervals Intervals
|
||||
}
|
||||
|
||||
func readTombstones(dir string) (TombstoneReader, SizeReader, error) {
|
||||
b, err := ioutil.ReadFile(filepath.Join(dir, tombstoneFilename))
|
||||
if os.IsNotExist(err) {
|
||||
return newMemTombstones(), nil, nil
|
||||
} else if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
sr := &TombstoneFile{
|
||||
size: int64(len(b)),
|
||||
}
|
||||
|
||||
if len(b) < 5 {
|
||||
return nil, sr, errors.Wrap(encoding.ErrInvalidSize, "tombstones header")
|
||||
}
|
||||
|
||||
d := &encoding.Decbuf{B: b[:len(b)-4]} // 4 for the checksum.
|
||||
if mg := d.Be32(); mg != MagicTombstone {
|
||||
return nil, sr, fmt.Errorf("invalid magic number %x", mg)
|
||||
}
|
||||
if flag := d.Byte(); flag != tombstoneFormatV1 {
|
||||
return nil, sr, fmt.Errorf("invalid tombstone format %x", flag)
|
||||
}
|
||||
|
||||
if d.Err() != nil {
|
||||
return nil, sr, d.Err()
|
||||
}
|
||||
|
||||
// Verify checksum.
|
||||
hash := newCRC32()
|
||||
if _, err := hash.Write(d.Get()); err != nil {
|
||||
return nil, sr, errors.Wrap(err, "write to hash")
|
||||
}
|
||||
if binary.BigEndian.Uint32(b[len(b)-4:]) != hash.Sum32() {
|
||||
return nil, sr, errors.New("checksum did not match")
|
||||
}
|
||||
|
||||
stonesMap := newMemTombstones()
|
||||
|
||||
for d.Len() > 0 {
|
||||
k := d.Uvarint64()
|
||||
mint := d.Varint64()
|
||||
maxt := d.Varint64()
|
||||
if d.Err() != nil {
|
||||
return nil, sr, d.Err()
|
||||
}
|
||||
|
||||
stonesMap.addInterval(k, Interval{mint, maxt})
|
||||
}
|
||||
|
||||
return stonesMap, sr, nil
|
||||
}
|
||||
|
||||
type memTombstones struct {
|
||||
intvlGroups map[uint64]Intervals
|
||||
mtx sync.RWMutex
|
||||
}
|
||||
|
||||
// newMemTombstones creates new in memory TombstoneReader
|
||||
// that allows adding new intervals.
|
||||
func newMemTombstones() *memTombstones {
|
||||
return &memTombstones{intvlGroups: make(map[uint64]Intervals)}
|
||||
}
|
||||
|
||||
func (t *memTombstones) Get(ref uint64) (Intervals, error) {
|
||||
t.mtx.RLock()
|
||||
defer t.mtx.RUnlock()
|
||||
return t.intvlGroups[ref], nil
|
||||
}
|
||||
|
||||
func (t *memTombstones) Iter(f func(uint64, Intervals) error) error {
|
||||
t.mtx.RLock()
|
||||
defer t.mtx.RUnlock()
|
||||
for ref, ivs := range t.intvlGroups {
|
||||
if err := f(ref, ivs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *memTombstones) Total() uint64 {
|
||||
t.mtx.RLock()
|
||||
defer t.mtx.RUnlock()
|
||||
|
||||
total := uint64(0)
|
||||
for _, ivs := range t.intvlGroups {
|
||||
total += uint64(len(ivs))
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// addInterval to an existing memTombstones
|
||||
func (t *memTombstones) addInterval(ref uint64, itvs ...Interval) {
|
||||
t.mtx.Lock()
|
||||
defer t.mtx.Unlock()
|
||||
for _, itv := range itvs {
|
||||
t.intvlGroups[ref] = t.intvlGroups[ref].add(itv)
|
||||
}
|
||||
}
|
||||
|
||||
// TombstoneFile holds information about the tombstone file.
|
||||
type TombstoneFile struct {
|
||||
size int64
|
||||
}
|
||||
|
||||
// Size returns the tombstone file size.
|
||||
func (t *TombstoneFile) Size() int64 {
|
||||
return t.size
|
||||
}
|
||||
|
||||
func (*memTombstones) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Interval represents a single time-interval.
|
||||
type Interval struct {
|
||||
Mint, Maxt int64
|
||||
}
|
||||
|
||||
func (tr Interval) inBounds(t int64) bool {
|
||||
return t >= tr.Mint && t <= tr.Maxt
|
||||
}
|
||||
|
||||
func (tr Interval) isSubrange(dranges Intervals) bool {
|
||||
for _, r := range dranges {
|
||||
if r.inBounds(tr.Mint) && r.inBounds(tr.Maxt) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Intervals represents a set of increasing and non-overlapping time-intervals.
|
||||
type Intervals []Interval
|
||||
|
||||
// add the new time-range to the existing ones.
|
||||
// The existing ones must be sorted.
|
||||
func (itvs Intervals) add(n Interval) Intervals {
|
||||
for i, r := range itvs {
|
||||
// TODO(gouthamve): Make this codepath easier to digest.
|
||||
if r.inBounds(n.Mint-1) || r.inBounds(n.Mint) {
|
||||
if n.Maxt > r.Maxt {
|
||||
itvs[i].Maxt = n.Maxt
|
||||
}
|
||||
|
||||
j := 0
|
||||
for _, r2 := range itvs[i+1:] {
|
||||
if n.Maxt < r2.Mint {
|
||||
break
|
||||
}
|
||||
j++
|
||||
}
|
||||
if j != 0 {
|
||||
if itvs[i+j].Maxt > n.Maxt {
|
||||
itvs[i].Maxt = itvs[i+j].Maxt
|
||||
}
|
||||
itvs = append(itvs[:i+1], itvs[i+j+1:]...)
|
||||
}
|
||||
return itvs
|
||||
}
|
||||
|
||||
if r.inBounds(n.Maxt+1) || r.inBounds(n.Maxt) {
|
||||
if n.Mint < r.Maxt {
|
||||
itvs[i].Mint = n.Mint
|
||||
}
|
||||
return itvs
|
||||
}
|
||||
|
||||
if n.Mint < r.Mint {
|
||||
newRange := make(Intervals, i, len(itvs[:i])+1)
|
||||
copy(newRange, itvs[:i])
|
||||
newRange = append(newRange, n)
|
||||
newRange = append(newRange, itvs[i:]...)
|
||||
|
||||
return newRange
|
||||
}
|
||||
}
|
||||
|
||||
itvs = append(itvs, n)
|
||||
return itvs
|
||||
}
|
1306
vendor/github.com/prometheus/tsdb/wal.go
generated
vendored
Normal file
1306
vendor/github.com/prometheus/tsdb/wal.go
generated
vendored
Normal file
File diff suppressed because it is too large
Load diff
284
vendor/github.com/prometheus/tsdb/wal/live_reader.go
generated
vendored
Normal file
284
vendor/github.com/prometheus/tsdb/wal/live_reader.go
generated
vendored
Normal file
|
@ -0,0 +1,284 @@
|
|||
// Copyright 2019 The Prometheus Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package wal
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
|
||||
"github.com/go-kit/kit/log"
|
||||
"github.com/go-kit/kit/log/level"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
readerCorruptionErrors = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "prometheus_tsdb_wal_reader_corruption_errors",
|
||||
Help: "Errors encountered when reading the WAL.",
|
||||
}, []string{"error"})
|
||||
)
|
||||
|
||||
// NewLiveReader returns a new live reader.
|
||||
func NewLiveReader(logger log.Logger, r io.Reader) *LiveReader {
|
||||
return &LiveReader{
|
||||
logger: logger,
|
||||
rdr: r,
|
||||
|
||||
// Until we understand how they come about, make readers permissive
|
||||
// to records spanning pages.
|
||||
permissive: true,
|
||||
}
|
||||
}
|
||||
|
||||
// LiveReader reads WAL records from an io.Reader. It allows reading of WALs
|
||||
// that are still in the process of being written, and returns records as soon
|
||||
// as they can be read.
|
||||
type LiveReader struct {
|
||||
logger log.Logger
|
||||
rdr io.Reader
|
||||
err error
|
||||
rec []byte
|
||||
hdr [recordHeaderSize]byte
|
||||
buf [pageSize]byte
|
||||
readIndex int // Index in buf to start at for next read.
|
||||
writeIndex int // Index in buf to start at for next write.
|
||||
total int64 // Total bytes processed during reading in calls to Next().
|
||||
index int // Used to track partial records, should be 0 at the start of every new record.
|
||||
|
||||
// For testing, we can treat EOF as a non-error.
|
||||
eofNonErr bool
|
||||
|
||||
// We sometime see records span page boundaries. Should never happen, but it
|
||||
// does. Until we track down why, set permissive to true to tolerate it.
|
||||
// NB the non-ive Reader implementation allows for this.
|
||||
permissive bool
|
||||
}
|
||||
|
||||
// Err returns any errors encountered reading the WAL. io.EOFs are not terminal
|
||||
// and Next can be tried again. Non-EOFs are terminal, and the reader should
|
||||
// not be used again. It is up to the user to decide when to stop trying should
|
||||
// io.EOF be returned.
|
||||
func (r *LiveReader) Err() error {
|
||||
if r.eofNonErr && r.err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return r.err
|
||||
}
|
||||
|
||||
// Offset returns the number of bytes consumed from this segment.
|
||||
func (r *LiveReader) Offset() int64 {
|
||||
return r.total
|
||||
}
|
||||
|
||||
func (r *LiveReader) fillBuffer() (int, error) {
|
||||
n, err := r.rdr.Read(r.buf[r.writeIndex:len(r.buf)])
|
||||
r.writeIndex += n
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Next returns true if Record() will contain a full record.
|
||||
// If Next returns false, you should always checked the contents of Error().
|
||||
// Return false guarantees there are no more records if the segment is closed
|
||||
// and not corrupt, otherwise if Err() == io.EOF you should try again when more
|
||||
// data has been written.
|
||||
func (r *LiveReader) Next() bool {
|
||||
for {
|
||||
// If buildRecord returns a non-EOF error, its game up - the segment is
|
||||
// corrupt. If buildRecord returns an EOF, we try and read more in
|
||||
// fillBuffer later on. If that fails to read anything (n=0 && err=EOF),
|
||||
// we return EOF and the user can try again later. If we have a full
|
||||
// page, buildRecord is guaranteed to return a record or a non-EOF; it
|
||||
// has checks the records fit in pages.
|
||||
if ok, err := r.buildRecord(); ok {
|
||||
return true
|
||||
} else if err != nil && err != io.EOF {
|
||||
r.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
// If we've filled the page and not found a record, this
|
||||
// means records have started to span pages. Shouldn't happen
|
||||
// but does and until we found out why, we need to deal with this.
|
||||
if r.permissive && r.writeIndex == pageSize && r.readIndex > 0 {
|
||||
copy(r.buf[:], r.buf[r.readIndex:])
|
||||
r.writeIndex -= r.readIndex
|
||||
r.readIndex = 0
|
||||
continue
|
||||
}
|
||||
|
||||
if r.readIndex == pageSize {
|
||||
r.writeIndex = 0
|
||||
r.readIndex = 0
|
||||
}
|
||||
|
||||
if r.writeIndex != pageSize {
|
||||
n, err := r.fillBuffer()
|
||||
if n == 0 || (err != nil && err != io.EOF) {
|
||||
r.err = err
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Record returns the current record.
|
||||
// The returned byte slice is only valid until the next call to Next.
|
||||
func (r *LiveReader) Record() []byte {
|
||||
return r.rec
|
||||
}
|
||||
|
||||
// Rebuild a full record from potentially partial records. Returns false
|
||||
// if there was an error or if we weren't able to read a record for any reason.
|
||||
// Returns true if we read a full record. Any record data is appended to
|
||||
// LiveReader.rec
|
||||
func (r *LiveReader) buildRecord() (bool, error) {
|
||||
for {
|
||||
// Check that we have data in the internal buffer to read.
|
||||
if r.writeIndex <= r.readIndex {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Attempt to read a record, partial or otherwise.
|
||||
temp, n, err := r.readRecord()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
r.readIndex += n
|
||||
r.total += int64(n)
|
||||
if temp == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
rt := recType(r.hdr[0])
|
||||
if rt == recFirst || rt == recFull {
|
||||
r.rec = r.rec[:0]
|
||||
}
|
||||
r.rec = append(r.rec, temp...)
|
||||
|
||||
if err := validateRecord(rt, r.index); err != nil {
|
||||
r.index = 0
|
||||
return false, err
|
||||
}
|
||||
if rt == recLast || rt == recFull {
|
||||
r.index = 0
|
||||
return true, nil
|
||||
}
|
||||
// Only increment i for non-zero records since we use it
|
||||
// to determine valid content record sequences.
|
||||
r.index++
|
||||
}
|
||||
}
|
||||
|
||||
// Returns an error if the recType and i indicate an invalid record sequence.
|
||||
// As an example, if i is > 0 because we've read some amount of a partial record
|
||||
// (recFirst, recMiddle, etc. but not recLast) and then we get another recFirst or recFull
|
||||
// instead of a recLast or recMiddle we would have an invalid record.
|
||||
func validateRecord(typ recType, i int) error {
|
||||
switch typ {
|
||||
case recFull:
|
||||
if i != 0 {
|
||||
return errors.New("unexpected full record")
|
||||
}
|
||||
return nil
|
||||
case recFirst:
|
||||
if i != 0 {
|
||||
return errors.New("unexpected first record, dropping buffer")
|
||||
}
|
||||
return nil
|
||||
case recMiddle:
|
||||
if i == 0 {
|
||||
return errors.New("unexpected middle record, dropping buffer")
|
||||
}
|
||||
return nil
|
||||
case recLast:
|
||||
if i == 0 {
|
||||
return errors.New("unexpected last record, dropping buffer")
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return errors.Errorf("unexpected record type %d", typ)
|
||||
}
|
||||
}
|
||||
|
||||
// Read a sub-record (see recType) from the buffer. It could potentially
|
||||
// be a full record (recFull) if the record fits within the bounds of a single page.
|
||||
// Returns a byte slice of the record data read, the number of bytes read, and an error
|
||||
// if there's a non-zero byte in a page term record or the record checksum fails.
|
||||
// This is a non-method function to make it clear it does not mutate the reader.
|
||||
func (r *LiveReader) readRecord() ([]byte, int, error) {
|
||||
// Special case: for recPageTerm, check that are all zeros to end of page,
|
||||
// consume them but don't return them.
|
||||
if r.buf[r.readIndex] == byte(recPageTerm) {
|
||||
// End of page won't necessarily be end of buffer, as we may have
|
||||
// got misaligned by records spanning page boundaries.
|
||||
// r.total % pageSize is the offset into the current page
|
||||
// that r.readIndex points to in buf. Therefore
|
||||
// pageSize - (r.total % pageSize) is the amount left to read of
|
||||
// the current page.
|
||||
remaining := int(pageSize - (r.total % pageSize))
|
||||
if r.readIndex+remaining > r.writeIndex {
|
||||
return nil, 0, io.EOF
|
||||
}
|
||||
|
||||
for i := r.readIndex; i < r.readIndex+remaining; i++ {
|
||||
if r.buf[i] != 0 {
|
||||
return nil, 0, errors.New("unexpected non-zero byte in page term bytes")
|
||||
}
|
||||
}
|
||||
|
||||
return nil, remaining, nil
|
||||
}
|
||||
|
||||
// Not a recPageTerm; read the record and check the checksum.
|
||||
if r.writeIndex-r.readIndex < recordHeaderSize {
|
||||
return nil, 0, io.EOF
|
||||
}
|
||||
|
||||
copy(r.hdr[:], r.buf[r.readIndex:r.readIndex+recordHeaderSize])
|
||||
length := int(binary.BigEndian.Uint16(r.hdr[1:]))
|
||||
crc := binary.BigEndian.Uint32(r.hdr[3:])
|
||||
if r.readIndex+recordHeaderSize+length > pageSize {
|
||||
if !r.permissive {
|
||||
return nil, 0, fmt.Errorf("record would overflow current page: %d > %d", r.readIndex+recordHeaderSize+length, pageSize)
|
||||
}
|
||||
readerCorruptionErrors.WithLabelValues("record_span_page").Inc()
|
||||
level.Warn(r.logger).Log("msg", "record spans page boundaries", "start", r.readIndex, "end", recordHeaderSize+length, "pageSize", pageSize)
|
||||
}
|
||||
if recordHeaderSize+length > pageSize {
|
||||
return nil, 0, fmt.Errorf("record length greater than a single page: %d > %d", recordHeaderSize+length, pageSize)
|
||||
}
|
||||
if r.readIndex+recordHeaderSize+length > r.writeIndex {
|
||||
return nil, 0, io.EOF
|
||||
}
|
||||
|
||||
rec := r.buf[r.readIndex+recordHeaderSize : r.readIndex+recordHeaderSize+length]
|
||||
if c := crc32.Checksum(rec, castagnoliTable); c != crc {
|
||||
return nil, 0, errors.Errorf("unexpected checksum %x, expected %x", c, crc)
|
||||
}
|
||||
|
||||
return rec, length + recordHeaderSize, nil
|
||||
}
|
||||
|
||||
func min(i, j int) int {
|
||||
if i < j {
|
||||
return i
|
||||
}
|
||||
return j
|
||||
}
|
183
vendor/github.com/prometheus/tsdb/wal/reader.go
generated
vendored
Normal file
183
vendor/github.com/prometheus/tsdb/wal/reader.go
generated
vendored
Normal file
|
@ -0,0 +1,183 @@
|
|||
// Copyright 2019 The Prometheus Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package wal
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Reader reads WAL records from an io.Reader.
|
||||
type Reader struct {
|
||||
rdr io.Reader
|
||||
err error
|
||||
rec []byte
|
||||
buf [pageSize]byte
|
||||
total int64 // Total bytes processed.
|
||||
curRecTyp recType // Used for checking that the last record is not torn.
|
||||
}
|
||||
|
||||
// NewReader returns a new reader.
|
||||
func NewReader(r io.Reader) *Reader {
|
||||
return &Reader{rdr: r}
|
||||
}
|
||||
|
||||
// Next advances the reader to the next records and returns true if it exists.
|
||||
// It must not be called again after it returned false.
|
||||
func (r *Reader) Next() bool {
|
||||
err := r.next()
|
||||
if errors.Cause(err) == io.EOF {
|
||||
// The last WAL segment record shouldn't be torn(should be full or last).
|
||||
// The last record would be torn after a crash just before
|
||||
// the last record part could be persisted to disk.
|
||||
if recType(r.curRecTyp) == recFirst || recType(r.curRecTyp) == recMiddle {
|
||||
r.err = errors.New("last record is torn")
|
||||
}
|
||||
return false
|
||||
}
|
||||
r.err = err
|
||||
return r.err == nil
|
||||
}
|
||||
|
||||
func (r *Reader) next() (err error) {
|
||||
// We have to use r.buf since allocating byte arrays here fails escape
|
||||
// analysis and ends up on the heap, even though it seemingly should not.
|
||||
hdr := r.buf[:recordHeaderSize]
|
||||
buf := r.buf[recordHeaderSize:]
|
||||
|
||||
r.rec = r.rec[:0]
|
||||
|
||||
i := 0
|
||||
for {
|
||||
if _, err = io.ReadFull(r.rdr, hdr[:1]); err != nil {
|
||||
return errors.Wrap(err, "read first header byte")
|
||||
}
|
||||
r.total++
|
||||
r.curRecTyp = recType(hdr[0])
|
||||
|
||||
// Gobble up zero bytes.
|
||||
if r.curRecTyp == recPageTerm {
|
||||
// recPageTerm is a single byte that indicates the rest of the page is padded.
|
||||
// If it's the first byte in a page, buf is too small and
|
||||
// needs to be resized to fit pageSize-1 bytes.
|
||||
buf = r.buf[1:]
|
||||
|
||||
// We are pedantic and check whether the zeros are actually up
|
||||
// to a page boundary.
|
||||
// It's not strictly necessary but may catch sketchy state early.
|
||||
k := pageSize - (r.total % pageSize)
|
||||
if k == pageSize {
|
||||
continue // Initial 0 byte was last page byte.
|
||||
}
|
||||
n, err := io.ReadFull(r.rdr, buf[:k])
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "read remaining zeros")
|
||||
}
|
||||
r.total += int64(n)
|
||||
|
||||
for _, c := range buf[:k] {
|
||||
if c != 0 {
|
||||
return errors.New("unexpected non-zero byte in padded page")
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
n, err := io.ReadFull(r.rdr, hdr[1:])
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "read remaining header")
|
||||
}
|
||||
r.total += int64(n)
|
||||
|
||||
var (
|
||||
length = binary.BigEndian.Uint16(hdr[1:])
|
||||
crc = binary.BigEndian.Uint32(hdr[3:])
|
||||
)
|
||||
|
||||
if length > pageSize-recordHeaderSize {
|
||||
return errors.Errorf("invalid record size %d", length)
|
||||
}
|
||||
n, err = io.ReadFull(r.rdr, buf[:length])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.total += int64(n)
|
||||
|
||||
if n != int(length) {
|
||||
return errors.Errorf("invalid size: expected %d, got %d", length, n)
|
||||
}
|
||||
if c := crc32.Checksum(buf[:length], castagnoliTable); c != crc {
|
||||
return errors.Errorf("unexpected checksum %x, expected %x", c, crc)
|
||||
}
|
||||
r.rec = append(r.rec, buf[:length]...)
|
||||
|
||||
if err := validateRecord(r.curRecTyp, i); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.curRecTyp == recLast || r.curRecTyp == recFull {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Only increment i for non-zero records since we use it
|
||||
// to determine valid content record sequences.
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
// Err returns the last encountered error wrapped in a corruption error.
|
||||
// If the reader does not allow to infer a segment index and offset, a total
|
||||
// offset in the reader stream will be provided.
|
||||
func (r *Reader) Err() error {
|
||||
if r.err == nil {
|
||||
return nil
|
||||
}
|
||||
if b, ok := r.rdr.(*segmentBufReader); ok {
|
||||
return &CorruptionErr{
|
||||
Err: r.err,
|
||||
Dir: b.segs[b.cur].Dir(),
|
||||
Segment: b.segs[b.cur].Index(),
|
||||
Offset: int64(b.off),
|
||||
}
|
||||
}
|
||||
return &CorruptionErr{
|
||||
Err: r.err,
|
||||
Segment: -1,
|
||||
Offset: r.total,
|
||||
}
|
||||
}
|
||||
|
||||
// Record returns the current record. The returned byte slice is only
|
||||
// valid until the next call to Next.
|
||||
func (r *Reader) Record() []byte {
|
||||
return r.rec
|
||||
}
|
||||
|
||||
// Segment returns the current segment being read.
|
||||
func (r *Reader) Segment() int {
|
||||
if b, ok := r.rdr.(*segmentBufReader); ok {
|
||||
return b.segs[b.cur].Index()
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// Offset returns the current position of the segment being read.
|
||||
func (r *Reader) Offset() int64 {
|
||||
if b, ok := r.rdr.(*segmentBufReader); ok {
|
||||
return int64(b.off)
|
||||
}
|
||||
return r.total
|
||||
}
|
764
vendor/github.com/prometheus/tsdb/wal/wal.go
generated
vendored
Normal file
764
vendor/github.com/prometheus/tsdb/wal/wal.go
generated
vendored
Normal file
|
@ -0,0 +1,764 @@
|
|||
// Copyright 2017 The Prometheus Authors
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package wal
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-kit/kit/log"
|
||||
"github.com/go-kit/kit/log/level"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/tsdb/fileutil"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultSegmentSize = 128 * 1024 * 1024 // 128 MB
|
||||
pageSize = 32 * 1024 // 32KB
|
||||
recordHeaderSize = 7
|
||||
)
|
||||
|
||||
// The table gets initialized with sync.Once but may still cause a race
|
||||
// with any other use of the crc32 package anywhere. Thus we initialize it
|
||||
// before.
|
||||
var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// page is an in memory buffer used to batch disk writes.
|
||||
// Records bigger than the page size are split and flushed separately.
|
||||
// A flush is triggered when a single records doesn't fit the page size or
|
||||
// when the next record can't fit in the remaining free page space.
|
||||
type page struct {
|
||||
alloc int
|
||||
flushed int
|
||||
buf [pageSize]byte
|
||||
}
|
||||
|
||||
func (p *page) remaining() int {
|
||||
return pageSize - p.alloc
|
||||
}
|
||||
|
||||
func (p *page) full() bool {
|
||||
return pageSize-p.alloc < recordHeaderSize
|
||||
}
|
||||
|
||||
// Segment represents a segment file.
|
||||
type Segment struct {
|
||||
*os.File
|
||||
dir string
|
||||
i int
|
||||
}
|
||||
|
||||
// Index returns the index of the segment.
|
||||
func (s *Segment) Index() int {
|
||||
return s.i
|
||||
}
|
||||
|
||||
// Dir returns the directory of the segment.
|
||||
func (s *Segment) Dir() string {
|
||||
return s.dir
|
||||
}
|
||||
|
||||
// CorruptionErr is an error that's returned when corruption is encountered.
|
||||
type CorruptionErr struct {
|
||||
Dir string
|
||||
Segment int
|
||||
Offset int64
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *CorruptionErr) Error() string {
|
||||
if e.Segment < 0 {
|
||||
return fmt.Sprintf("corruption after %d bytes: %s", e.Offset, e.Err)
|
||||
}
|
||||
return fmt.Sprintf("corruption in segment %s at %d: %s", SegmentName(e.Dir, e.Segment), e.Offset, e.Err)
|
||||
}
|
||||
|
||||
// OpenWriteSegment opens segment k in dir. The returned segment is ready for new appends.
|
||||
func OpenWriteSegment(logger log.Logger, dir string, k int) (*Segment, error) {
|
||||
segName := SegmentName(dir, k)
|
||||
f, err := os.OpenFile(segName, os.O_WRONLY|os.O_APPEND, 0666)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
// If the last page is torn, fill it with zeros.
|
||||
// In case it was torn after all records were written successfully, this
|
||||
// will just pad the page and everything will be fine.
|
||||
// If it was torn mid-record, a full read (which the caller should do anyway
|
||||
// to ensure integrity) will detect it as a corruption by the end.
|
||||
if d := stat.Size() % pageSize; d != 0 {
|
||||
level.Warn(logger).Log("msg", "last page of the wal is torn, filling it with zeros", "segment", segName)
|
||||
if _, err := f.Write(make([]byte, pageSize-d)); err != nil {
|
||||
f.Close()
|
||||
return nil, errors.Wrap(err, "zero-pad torn page")
|
||||
}
|
||||
}
|
||||
return &Segment{File: f, i: k, dir: dir}, nil
|
||||
}
|
||||
|
||||
// CreateSegment creates a new segment k in dir.
|
||||
func CreateSegment(dir string, k int) (*Segment, error) {
|
||||
f, err := os.OpenFile(SegmentName(dir, k), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Segment{File: f, i: k, dir: dir}, nil
|
||||
}
|
||||
|
||||
// OpenReadSegment opens the segment with the given filename.
|
||||
func OpenReadSegment(fn string) (*Segment, error) {
|
||||
k, err := strconv.Atoi(filepath.Base(fn))
|
||||
if err != nil {
|
||||
return nil, errors.New("not a valid filename")
|
||||
}
|
||||
f, err := os.Open(fn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Segment{File: f, i: k, dir: filepath.Dir(fn)}, nil
|
||||
}
|
||||
|
||||
// WAL is a write ahead log that stores records in segment files.
|
||||
// It must be read from start to end once before logging new data.
|
||||
// If an error occurs during read, the repair procedure must be called
|
||||
// before it's safe to do further writes.
|
||||
//
|
||||
// Segments are written to in pages of 32KB, with records possibly split
|
||||
// across page boundaries.
|
||||
// Records are never split across segments to allow full segments to be
|
||||
// safely truncated. It also ensures that torn writes never corrupt records
|
||||
// beyond the most recent segment.
|
||||
type WAL struct {
|
||||
dir string
|
||||
logger log.Logger
|
||||
segmentSize int
|
||||
mtx sync.RWMutex
|
||||
segment *Segment // active segment
|
||||
donePages int // pages written to the segment
|
||||
page *page // active page
|
||||
stopc chan chan struct{}
|
||||
actorc chan func()
|
||||
closed bool // To allow calling Close() more than once without blocking.
|
||||
|
||||
fsyncDuration prometheus.Summary
|
||||
pageFlushes prometheus.Counter
|
||||
pageCompletions prometheus.Counter
|
||||
truncateFail prometheus.Counter
|
||||
truncateTotal prometheus.Counter
|
||||
}
|
||||
|
||||
// New returns a new WAL over the given directory.
|
||||
func New(logger log.Logger, reg prometheus.Registerer, dir string) (*WAL, error) {
|
||||
return NewSize(logger, reg, dir, DefaultSegmentSize)
|
||||
}
|
||||
|
||||
// NewSize returns a new WAL over the given directory.
|
||||
// New segments are created with the specified size.
|
||||
func NewSize(logger log.Logger, reg prometheus.Registerer, dir string, segmentSize int) (*WAL, error) {
|
||||
if segmentSize%pageSize != 0 {
|
||||
return nil, errors.New("invalid segment size")
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0777); err != nil {
|
||||
return nil, errors.Wrap(err, "create dir")
|
||||
}
|
||||
if logger == nil {
|
||||
logger = log.NewNopLogger()
|
||||
}
|
||||
w := &WAL{
|
||||
dir: dir,
|
||||
logger: logger,
|
||||
segmentSize: segmentSize,
|
||||
page: &page{},
|
||||
actorc: make(chan func(), 100),
|
||||
stopc: make(chan chan struct{}),
|
||||
}
|
||||
w.fsyncDuration = prometheus.NewSummary(prometheus.SummaryOpts{
|
||||
Name: "prometheus_tsdb_wal_fsync_duration_seconds",
|
||||
Help: "Duration of WAL fsync.",
|
||||
})
|
||||
w.pageFlushes = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "prometheus_tsdb_wal_page_flushes_total",
|
||||
Help: "Total number of page flushes.",
|
||||
})
|
||||
w.pageCompletions = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "prometheus_tsdb_wal_completed_pages_total",
|
||||
Help: "Total number of completed pages.",
|
||||
})
|
||||
w.truncateFail = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "prometheus_tsdb_wal_truncations_failed_total",
|
||||
Help: "Total number of WAL truncations that failed.",
|
||||
})
|
||||
w.truncateTotal = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "prometheus_tsdb_wal_truncations_total",
|
||||
Help: "Total number of WAL truncations attempted.",
|
||||
})
|
||||
if reg != nil {
|
||||
reg.MustRegister(w.fsyncDuration, w.pageFlushes, w.pageCompletions, w.truncateFail, w.truncateTotal)
|
||||
}
|
||||
|
||||
_, j, err := w.Segments()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "get segment range")
|
||||
}
|
||||
// Fresh dir, no segments yet.
|
||||
if j == -1 {
|
||||
segment, err := CreateSegment(w.dir, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := w.setSegment(segment); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
segment, err := OpenWriteSegment(logger, w.dir, j)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := w.setSegment(segment); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
go w.run()
|
||||
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Dir returns the directory of the WAL.
|
||||
func (w *WAL) Dir() string {
|
||||
return w.dir
|
||||
}
|
||||
|
||||
func (w *WAL) run() {
|
||||
Loop:
|
||||
for {
|
||||
select {
|
||||
case f := <-w.actorc:
|
||||
f()
|
||||
case donec := <-w.stopc:
|
||||
close(w.actorc)
|
||||
defer close(donec)
|
||||
break Loop
|
||||
}
|
||||
}
|
||||
// Drain and process any remaining functions.
|
||||
for f := range w.actorc {
|
||||
f()
|
||||
}
|
||||
}
|
||||
|
||||
// Repair attempts to repair the WAL based on the error.
|
||||
// It discards all data after the corruption.
|
||||
func (w *WAL) Repair(origErr error) error {
|
||||
// We could probably have a mode that only discards torn records right around
|
||||
// the corruption to preserve as data much as possible.
|
||||
// But that's not generally applicable if the records have any kind of causality.
|
||||
// Maybe as an extra mode in the future if mid-WAL corruptions become
|
||||
// a frequent concern.
|
||||
err := errors.Cause(origErr) // So that we can pick up errors even if wrapped.
|
||||
|
||||
cerr, ok := err.(*CorruptionErr)
|
||||
if !ok {
|
||||
return errors.Wrap(origErr, "cannot handle error")
|
||||
}
|
||||
if cerr.Segment < 0 {
|
||||
return errors.New("corruption error does not specify position")
|
||||
}
|
||||
|
||||
level.Warn(w.logger).Log("msg", "starting corruption repair",
|
||||
"segment", cerr.Segment, "offset", cerr.Offset)
|
||||
|
||||
// All segments behind the corruption can no longer be used.
|
||||
segs, err := listSegments(w.dir)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "list segments")
|
||||
}
|
||||
level.Warn(w.logger).Log("msg", "deleting all segments newer than corrupted segment", "segment", cerr.Segment)
|
||||
|
||||
for _, s := range segs {
|
||||
if w.segment.i == s.index {
|
||||
// The active segment needs to be removed,
|
||||
// close it first (Windows!). Can be closed safely
|
||||
// as we set the current segment to repaired file
|
||||
// below.
|
||||
if err := w.segment.Close(); err != nil {
|
||||
return errors.Wrap(err, "close active segment")
|
||||
}
|
||||
}
|
||||
if s.index <= cerr.Segment {
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(filepath.Join(w.dir, s.name)); err != nil {
|
||||
return errors.Wrapf(err, "delete segment:%v", s.index)
|
||||
}
|
||||
}
|
||||
// Regardless of the corruption offset, no record reaches into the previous segment.
|
||||
// So we can safely repair the WAL by removing the segment and re-inserting all
|
||||
// its records up to the corruption.
|
||||
level.Warn(w.logger).Log("msg", "rewrite corrupted segment", "segment", cerr.Segment)
|
||||
|
||||
fn := SegmentName(w.dir, cerr.Segment)
|
||||
tmpfn := fn + ".repair"
|
||||
|
||||
if err := fileutil.Rename(fn, tmpfn); err != nil {
|
||||
return err
|
||||
}
|
||||
// Create a clean segment and make it the active one.
|
||||
s, err := CreateSegment(w.dir, cerr.Segment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.setSegment(s); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f, err := os.Open(tmpfn)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "open segment")
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
r := NewReader(bufio.NewReader(f))
|
||||
|
||||
for r.Next() {
|
||||
// Add records only up to the where the error was.
|
||||
if r.Offset() >= cerr.Offset {
|
||||
break
|
||||
}
|
||||
if err := w.Log(r.Record()); err != nil {
|
||||
return errors.Wrap(err, "insert record")
|
||||
}
|
||||
}
|
||||
// We expect an error here from r.Err(), so nothing to handle.
|
||||
|
||||
// We explicitly close even when there is a defer for Windows to be
|
||||
// able to delete it. The defer is in place to close it in-case there
|
||||
// are errors above.
|
||||
if err := f.Close(); err != nil {
|
||||
return errors.Wrap(err, "close corrupted file")
|
||||
}
|
||||
if err := os.Remove(tmpfn); err != nil {
|
||||
return errors.Wrap(err, "delete corrupted segment")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SegmentName builds a segment name for the directory.
|
||||
func SegmentName(dir string, i int) string {
|
||||
return filepath.Join(dir, fmt.Sprintf("%08d", i))
|
||||
}
|
||||
|
||||
// nextSegment creates the next segment and closes the previous one.
|
||||
func (w *WAL) nextSegment() error {
|
||||
// Only flush the current page if it actually holds data.
|
||||
if w.page.alloc > 0 {
|
||||
if err := w.flushPage(true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
next, err := CreateSegment(w.dir, w.segment.Index()+1)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create new segment file")
|
||||
}
|
||||
prev := w.segment
|
||||
if err := w.setSegment(next); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Don't block further writes by fsyncing the last segment.
|
||||
w.actorc <- func() {
|
||||
if err := w.fsync(prev); err != nil {
|
||||
level.Error(w.logger).Log("msg", "sync previous segment", "err", err)
|
||||
}
|
||||
if err := prev.Close(); err != nil {
|
||||
level.Error(w.logger).Log("msg", "close previous segment", "err", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *WAL) setSegment(segment *Segment) error {
|
||||
w.segment = segment
|
||||
|
||||
// Correctly initialize donePages.
|
||||
stat, err := segment.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.donePages = int(stat.Size() / pageSize)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// flushPage writes the new contents of the page to disk. If no more records will fit into
|
||||
// the page, the remaining bytes will be set to zero and a new page will be started.
|
||||
// If clear is true, this is enforced regardless of how many bytes are left in the page.
|
||||
func (w *WAL) flushPage(clear bool) error {
|
||||
w.pageFlushes.Inc()
|
||||
|
||||
p := w.page
|
||||
clear = clear || p.full()
|
||||
|
||||
// No more data will fit into the page. Enqueue and clear it.
|
||||
if clear {
|
||||
p.alloc = pageSize // Write till end of page.
|
||||
w.pageCompletions.Inc()
|
||||
}
|
||||
n, err := w.segment.Write(p.buf[p.flushed:p.alloc])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.flushed += n
|
||||
|
||||
// We flushed an entire page, prepare a new one.
|
||||
if clear {
|
||||
for i := range p.buf {
|
||||
p.buf[i] = 0
|
||||
}
|
||||
p.alloc = 0
|
||||
p.flushed = 0
|
||||
w.donePages++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type recType uint8
|
||||
|
||||
const (
|
||||
recPageTerm recType = 0 // Rest of page is empty.
|
||||
recFull recType = 1 // Full record.
|
||||
recFirst recType = 2 // First fragment of a record.
|
||||
recMiddle recType = 3 // Middle fragments of a record.
|
||||
recLast recType = 4 // Final fragment of a record.
|
||||
)
|
||||
|
||||
func (t recType) String() string {
|
||||
switch t {
|
||||
case recPageTerm:
|
||||
return "zero"
|
||||
case recFull:
|
||||
return "full"
|
||||
case recFirst:
|
||||
return "first"
|
||||
case recMiddle:
|
||||
return "middle"
|
||||
case recLast:
|
||||
return "last"
|
||||
default:
|
||||
return "<invalid>"
|
||||
}
|
||||
}
|
||||
|
||||
func (w *WAL) pagesPerSegment() int {
|
||||
return w.segmentSize / pageSize
|
||||
}
|
||||
|
||||
// Log writes the records into the log.
|
||||
// Multiple records can be passed at once to reduce writes and increase throughput.
|
||||
func (w *WAL) Log(recs ...[]byte) error {
|
||||
w.mtx.Lock()
|
||||
defer w.mtx.Unlock()
|
||||
// Callers could just implement their own list record format but adding
|
||||
// a bit of extra logic here frees them from that overhead.
|
||||
for i, r := range recs {
|
||||
if err := w.log(r, i == len(recs)-1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// log writes rec to the log and forces a flush of the current page if its
|
||||
// the final record of a batch, the record is bigger than the page size or
|
||||
// the current page is full.
|
||||
func (w *WAL) log(rec []byte, final bool) error {
|
||||
// If the record is too big to fit within the active page in the current
|
||||
// segment, terminate the active segment and advance to the next one.
|
||||
// This ensures that records do not cross segment boundaries.
|
||||
left := w.page.remaining() - recordHeaderSize // Free space in the active page.
|
||||
left += (pageSize - recordHeaderSize) * (w.pagesPerSegment() - w.donePages - 1) // Free pages in the active segment.
|
||||
|
||||
if len(rec) > left {
|
||||
if err := w.nextSegment(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Populate as many pages as necessary to fit the record.
|
||||
// Be careful to always do one pass to ensure we write zero-length records.
|
||||
for i := 0; i == 0 || len(rec) > 0; i++ {
|
||||
p := w.page
|
||||
|
||||
// Find how much of the record we can fit into the page.
|
||||
var (
|
||||
l = min(len(rec), (pageSize-p.alloc)-recordHeaderSize)
|
||||
part = rec[:l]
|
||||
buf = p.buf[p.alloc:]
|
||||
typ recType
|
||||
)
|
||||
|
||||
switch {
|
||||
case i == 0 && len(part) == len(rec):
|
||||
typ = recFull
|
||||
case len(part) == len(rec):
|
||||
typ = recLast
|
||||
case i == 0:
|
||||
typ = recFirst
|
||||
default:
|
||||
typ = recMiddle
|
||||
}
|
||||
|
||||
buf[0] = byte(typ)
|
||||
crc := crc32.Checksum(part, castagnoliTable)
|
||||
binary.BigEndian.PutUint16(buf[1:], uint16(len(part)))
|
||||
binary.BigEndian.PutUint32(buf[3:], crc)
|
||||
|
||||
copy(buf[recordHeaderSize:], part)
|
||||
p.alloc += len(part) + recordHeaderSize
|
||||
|
||||
// By definition when a record is split it means its size is bigger than
|
||||
// the page boundary so the current page would be full and needs to be flushed.
|
||||
// On contrary if we wrote a full record, we can fit more records of the batch
|
||||
// into the page before flushing it.
|
||||
if final || typ != recFull || w.page.full() {
|
||||
if err := w.flushPage(false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
rec = rec[l:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Segments returns the range [first, n] of currently existing segments.
|
||||
// If no segments are found, first and n are -1.
|
||||
func (w *WAL) Segments() (first, last int, err error) {
|
||||
refs, err := listSegments(w.dir)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if len(refs) == 0 {
|
||||
return -1, -1, nil
|
||||
}
|
||||
return refs[0].index, refs[len(refs)-1].index, nil
|
||||
}
|
||||
|
||||
// Truncate drops all segments before i.
|
||||
func (w *WAL) Truncate(i int) (err error) {
|
||||
w.truncateTotal.Inc()
|
||||
defer func() {
|
||||
if err != nil {
|
||||
w.truncateFail.Inc()
|
||||
}
|
||||
}()
|
||||
refs, err := listSegments(w.dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, r := range refs {
|
||||
if r.index >= i {
|
||||
break
|
||||
}
|
||||
if err = os.Remove(filepath.Join(w.dir, r.name)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *WAL) fsync(f *Segment) error {
|
||||
start := time.Now()
|
||||
err := fileutil.Fsync(f.File)
|
||||
w.fsyncDuration.Observe(time.Since(start).Seconds())
|
||||
return err
|
||||
}
|
||||
|
||||
// Close flushes all writes and closes active segment.
|
||||
func (w *WAL) Close() (err error) {
|
||||
w.mtx.Lock()
|
||||
defer w.mtx.Unlock()
|
||||
|
||||
if w.closed {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush the last page and zero out all its remaining size.
|
||||
// We must not flush an empty page as it would falsely signal
|
||||
// the segment is done if we start writing to it again after opening.
|
||||
if w.page.alloc > 0 {
|
||||
if err := w.flushPage(true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
donec := make(chan struct{})
|
||||
w.stopc <- donec
|
||||
<-donec
|
||||
|
||||
if err = w.fsync(w.segment); err != nil {
|
||||
level.Error(w.logger).Log("msg", "sync previous segment", "err", err)
|
||||
}
|
||||
if err := w.segment.Close(); err != nil {
|
||||
level.Error(w.logger).Log("msg", "close previous segment", "err", err)
|
||||
}
|
||||
w.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
type segmentRef struct {
|
||||
name string
|
||||
index int
|
||||
}
|
||||
|
||||
func listSegments(dir string) (refs []segmentRef, err error) {
|
||||
files, err := fileutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var last int
|
||||
for _, fn := range files {
|
||||
k, err := strconv.Atoi(fn)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if len(refs) > 0 && k > last+1 {
|
||||
return nil, errors.New("segments are not sequential")
|
||||
}
|
||||
refs = append(refs, segmentRef{name: fn, index: k})
|
||||
last = k
|
||||
}
|
||||
sort.Slice(refs, func(i, j int) bool {
|
||||
return refs[i].index < refs[j].index
|
||||
})
|
||||
return refs, nil
|
||||
}
|
||||
|
||||
// SegmentRange groups segments by the directory and the first and last index it includes.
|
||||
type SegmentRange struct {
|
||||
Dir string
|
||||
First, Last int
|
||||
}
|
||||
|
||||
// NewSegmentsReader returns a new reader over all segments in the directory.
|
||||
func NewSegmentsReader(dir string) (io.ReadCloser, error) {
|
||||
return NewSegmentsRangeReader(SegmentRange{dir, -1, -1})
|
||||
}
|
||||
|
||||
// NewSegmentsRangeReader returns a new reader over the given WAL segment ranges.
|
||||
// If first or last are -1, the range is open on the respective end.
|
||||
func NewSegmentsRangeReader(sr ...SegmentRange) (io.ReadCloser, error) {
|
||||
var segs []*Segment
|
||||
|
||||
for _, sgmRange := range sr {
|
||||
refs, err := listSegments(sgmRange.Dir)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "list segment in dir:%v", sgmRange.Dir)
|
||||
}
|
||||
|
||||
for _, r := range refs {
|
||||
if sgmRange.First >= 0 && r.index < sgmRange.First {
|
||||
continue
|
||||
}
|
||||
if sgmRange.Last >= 0 && r.index > sgmRange.Last {
|
||||
break
|
||||
}
|
||||
s, err := OpenReadSegment(filepath.Join(sgmRange.Dir, r.name))
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "open segment:%v in dir:%v", r.name, sgmRange.Dir)
|
||||
}
|
||||
segs = append(segs, s)
|
||||
}
|
||||
}
|
||||
return newSegmentBufReader(segs...), nil
|
||||
}
|
||||
|
||||
// segmentBufReader is a buffered reader that reads in multiples of pages.
|
||||
// The main purpose is that we are able to track segment and offset for
|
||||
// corruption reporting. We have to be careful not to increment curr too
|
||||
// early, as it is used by Reader.Err() to tell Repair which segment is corrupt.
|
||||
// As such we pad the end of non-page align segments with zeros.
|
||||
type segmentBufReader struct {
|
||||
buf *bufio.Reader
|
||||
segs []*Segment
|
||||
cur int // Index into segs.
|
||||
off int // Offset of read data into current segment.
|
||||
}
|
||||
|
||||
func newSegmentBufReader(segs ...*Segment) *segmentBufReader {
|
||||
return &segmentBufReader{
|
||||
buf: bufio.NewReaderSize(segs[0], 16*pageSize),
|
||||
segs: segs,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *segmentBufReader) Close() (err error) {
|
||||
for _, s := range r.segs {
|
||||
if e := s.Close(); e != nil {
|
||||
err = e
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Read implements io.Reader.
|
||||
func (r *segmentBufReader) Read(b []byte) (n int, err error) {
|
||||
n, err = r.buf.Read(b)
|
||||
r.off += n
|
||||
|
||||
// If we succeeded, or hit a non-EOF, we can stop.
|
||||
if err == nil || err != io.EOF {
|
||||
return n, err
|
||||
}
|
||||
|
||||
// We hit EOF; fake out zero padding at the end of short segments, so we
|
||||
// don't increment curr too early and report the wrong segment as corrupt.
|
||||
if r.off%pageSize != 0 {
|
||||
i := 0
|
||||
for ; n+i < len(b) && (r.off+i)%pageSize != 0; i++ {
|
||||
b[n+i] = 0
|
||||
}
|
||||
|
||||
// Return early, even if we didn't fill b.
|
||||
r.off += i
|
||||
return n + i, nil
|
||||
}
|
||||
|
||||
// There is no more deta left in the curr segment and there are no more
|
||||
// segments left. Return EOF.
|
||||
if r.cur+1 >= len(r.segs) {
|
||||
return n, io.EOF
|
||||
}
|
||||
|
||||
// Move to next segment.
|
||||
r.cur++
|
||||
r.off = 0
|
||||
r.buf.Reset(r.segs[r.cur])
|
||||
return n, nil
|
||||
}
|
24
vendor/modules.txt
vendored
24
vendor/modules.txt
vendored
|
@ -18,6 +18,8 @@ github.com/brancz/gojsontoyaml
|
|||
# github.com/campoy/embedmd v1.0.0
|
||||
github.com/campoy/embedmd
|
||||
github.com/campoy/embedmd/embedmd
|
||||
# github.com/cespare/xxhash v1.1.0
|
||||
github.com/cespare/xxhash
|
||||
# github.com/davecgh/go-spew v1.1.1
|
||||
github.com/davecgh/go-spew/spew
|
||||
# github.com/emicklei/go-restful v2.6.0+incompatible
|
||||
|
@ -34,7 +36,7 @@ github.com/ghodss/yaml
|
|||
# github.com/go-kit/kit v0.8.0
|
||||
github.com/go-kit/kit/log
|
||||
github.com/go-kit/kit/log/level
|
||||
# github.com/go-logfmt/logfmt v0.3.0
|
||||
# github.com/go-logfmt/logfmt v0.4.0
|
||||
github.com/go-logfmt/logfmt
|
||||
# github.com/go-openapi/jsonpointer v0.19.0
|
||||
github.com/go-openapi/jsonpointer
|
||||
|
@ -59,14 +61,14 @@ github.com/google/gofuzz
|
|||
github.com/googleapis/gnostic/OpenAPIv2
|
||||
github.com/googleapis/gnostic/compiler
|
||||
github.com/googleapis/gnostic/extensions
|
||||
# github.com/hashicorp/go-version v0.0.0-20171129150820-4fe82ae3040f
|
||||
# github.com/hashicorp/go-version v1.1.0
|
||||
github.com/hashicorp/go-version
|
||||
# github.com/hashicorp/golang-lru v0.5.0
|
||||
# github.com/hashicorp/golang-lru v0.5.1
|
||||
github.com/hashicorp/golang-lru
|
||||
github.com/hashicorp/golang-lru/simplelru
|
||||
# github.com/imdario/mergo v0.3.5
|
||||
github.com/imdario/mergo
|
||||
# github.com/improbable-eng/thanos v0.3.2
|
||||
# github.com/improbable-eng/thanos v0.4.0
|
||||
github.com/improbable-eng/thanos/pkg/reloader
|
||||
github.com/improbable-eng/thanos/pkg/runutil
|
||||
# github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3
|
||||
|
@ -98,14 +100,17 @@ github.com/modern-go/concurrent
|
|||
github.com/modern-go/reflect2
|
||||
# github.com/oklog/run v1.0.0
|
||||
github.com/oklog/run
|
||||
# github.com/oklog/ulid v1.3.1
|
||||
github.com/oklog/ulid
|
||||
# github.com/pkg/errors v0.8.1
|
||||
github.com/pkg/errors
|
||||
# github.com/pmezard/go-difflib v1.0.0
|
||||
github.com/pmezard/go-difflib/difflib
|
||||
# github.com/prometheus/client_golang v0.9.1
|
||||
# github.com/prometheus/client_golang v0.9.2
|
||||
github.com/prometheus/client_golang/prometheus
|
||||
github.com/prometheus/client_golang/prometheus/promhttp
|
||||
github.com/prometheus/client_golang/prometheus/internal
|
||||
github.com/prometheus/client_golang/prometheus/promauto
|
||||
# github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910
|
||||
github.com/prometheus/client_model/go
|
||||
# github.com/prometheus/common v0.0.0-20181218105931-67670fe90761
|
||||
|
@ -117,6 +122,15 @@ github.com/prometheus/procfs
|
|||
github.com/prometheus/procfs/nfs
|
||||
github.com/prometheus/procfs/xfs
|
||||
github.com/prometheus/procfs/internal/util
|
||||
# github.com/prometheus/tsdb v0.6.1
|
||||
github.com/prometheus/tsdb
|
||||
github.com/prometheus/tsdb/chunkenc
|
||||
github.com/prometheus/tsdb/chunks
|
||||
github.com/prometheus/tsdb/encoding
|
||||
github.com/prometheus/tsdb/fileutil
|
||||
github.com/prometheus/tsdb/index
|
||||
github.com/prometheus/tsdb/labels
|
||||
github.com/prometheus/tsdb/wal
|
||||
# github.com/spf13/pflag v1.0.3
|
||||
github.com/spf13/pflag
|
||||
# github.com/stretchr/testify v1.3.0
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue