Skip to content

Commit af19d12

Browse files
Collector Synchronization (#184)
Signed-off-by: Anders Swanson <anders.swanson@oracle.com>
1 parent dfb9ca5 commit af19d12

File tree

2 files changed

+64
-49
lines changed

2 files changed

+64
-49
lines changed

‎collector/collector.go

Lines changed: 44 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -214,51 +214,26 @@ func (e *Exporter) scheduledScrape(tick *time.Time) {
214214

215215
func (e *Exporter) scrape(ch chan<- prometheus.Metric, tick *time.Time) {
216216
e.totalScrapes.Inc()
217-
var err error
218-
var scrapemutex sync.Mutex
219-
errChan := make(chan ScrapeResult, len(e.metricsToScrape.Metric))
217+
errChan := make(chan error, len(e.metricsToScrape.Metric))
218+
begun := time.Now()
220219

221-
defer func(begun time.Time) {
222-
// other error
223-
e.duration.Set(time.Since(begun).Seconds())
224-
if err == nil {
225-
e.error.Set(0)
226-
} else {
227-
e.error.Set(1)
228-
}
229-
230-
// scrape error
231-
close(errChan)
232-
for scrape := range errChan {
233-
if scrape.Err != nil {
234-
if shouldLogScrapeError(scrape.Err, scrape.Metric.IgnoreZeroResult) {
235-
level.Error(e.logger).Log("msg", "Error scraping metric",
236-
"Context", scrape.Metric.Context,
237-
"MetricsDesc", fmt.Sprint(scrape.Metric.MetricsDesc),
238-
"time", time.Since(scrape.ScrapeStart),
239-
"error", scrape.Err)
240-
}
241-
e.scrapeErrors.WithLabelValues(scrape.Metric.Context).Inc()
242-
}
243-
}
244-
245-
}(time.Now())
246-
247-
if err = e.db.Ping(); err != nil {
248-
level.Debug(e.logger).Log("msg", "error = "+err.Error())
249-
if strings.Contains(err.Error(), "sql: database is closed") {
220+
if connectionError := e.db.Ping(); connectionError != nil {
221+
level.Debug(e.logger).Log("msg", "error = "+connectionError.Error())
222+
if strings.Contains(connectionError.Error(), "sql: database is closed") {
250223
level.Info(e.logger).Log("msg", "Reconnecting to DB")
251-
err = e.connect()
252-
if err != nil {
253-
level.Error(e.logger).Log("msg", "Error reconnecting to DB", err)
224+
connectionError = e.connect()
225+
if connectionError != nil {
226+
level.Error(e.logger).Log("msg", "Error reconnecting to DB", connectionError)
254227
}
255228
}
256229
}
257230

258-
if err = e.db.Ping(); err != nil {
231+
if pingError := e.db.Ping(); pingError != nil {
259232
level.Error(e.logger).Log("msg", "Error pinging oracle",
260-
"error", err)
233+
"error", pingError)
261234
e.up.Set(0)
235+
e.error.Set(1)
236+
e.duration.Set(time.Since(begun).Seconds())
262237
return
263238
}
264239

@@ -271,15 +246,10 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric, tick *time.Time) {
271246
e.reloadMetrics()
272247
}
273248

274-
wg := sync.WaitGroup{}
275-
276249
for _, metric := range e.metricsToScrape.Metric {
277-
wg.Add(1)
278250
metric := metric //https://golang.org/doc/faq#closures_and_goroutines
279251

280252
go func() {
281-
defer wg.Done()
282-
283253
level.Debug(e.logger).Log("msg", "About to scrape metric",
284254
"Context", metric.Context,
285255
"MetricsDesc", fmt.Sprint(metric.MetricsDesc),
@@ -291,11 +261,13 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric, tick *time.Time) {
291261
"Request", metric.Request)
292262

293263
if len(metric.Request) == 0 {
264+
errChan <- errors.New("scrape request not found")
294265
level.Error(e.logger).Log("msg", "Error scraping for "+fmt.Sprint(metric.MetricsDesc)+". Did you forget to define request in your toml file?")
295266
return
296267
}
297268

298269
if len(metric.MetricsDesc) == 0 {
270+
errChan <- errors.New("metricsdesc not found")
299271
level.Error(e.logger).Log("msg", "Error scraping for query"+fmt.Sprint(metric.Request)+". Did you forget to define metricsdesc in your toml file?")
300272
return
301273
}
@@ -304,19 +276,26 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric, tick *time.Time) {
304276
if metricType == "histogram" {
305277
_, ok := metric.MetricsBuckets[column]
306278
if !ok {
279+
errChan <- errors.New("metricsbuckets not found")
307280
level.Error(e.logger).Log("msg", "Unable to find MetricsBuckets configuration key for metric. (metric="+column+")")
308281
return
309282
}
310283
}
311284
}
312285

313286
scrapeStart := time.Now()
314-
if err1 := func() error {
315-
scrapemutex.Lock()
316-
defer scrapemutex.Unlock()
317-
return e.ScrapeMetric(e.db, ch, metric, tick)
318-
}(); err1 != nil {
319-
errChan <- ScrapeResult{Err: err1, Metric: metric, ScrapeStart: scrapeStart}
287+
scrapeError := e.ScrapeMetric(e.db, ch, metric, tick)
288+
// Always send the scrapeError, nil or non-nil
289+
errChan <- scrapeError
290+
if scrapeError != nil {
291+
if shouldLogScrapeError(scrapeError, metric.IgnoreZeroResult) {
292+
level.Error(e.logger).Log("msg", "Error scraping metric",
293+
"Context", metric.Context,
294+
"MetricsDesc", fmt.Sprint(metric.MetricsDesc),
295+
"time", time.Since(scrapeStart),
296+
"error", scrapeError)
297+
}
298+
e.scrapeErrors.WithLabelValues(metric.Context).Inc()
320299
} else {
321300
level.Debug(e.logger).Log("msg", "Successfully scraped metric",
322301
"Context", metric.Context,
@@ -325,7 +304,23 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric, tick *time.Time) {
325304
}
326305
}()
327306
}
328-
wg.Wait()
307+
308+
e.afterScrape(begun, len(e.metricsToScrape.Metric), errChan)
309+
}
310+
311+
func (e *Exporter) afterScrape(begun time.Time, countMetrics int, errChan chan error) {
312+
// Receive all scrape errors
313+
totalErrors := 0.0
314+
for i := 0; i < countMetrics; i++ {
315+
scrapeError := <-errChan
316+
if scrapeError != nil {
317+
totalErrors++
318+
}
319+
}
320+
close(errChan)
321+
322+
e.duration.Set(time.Since(begun).Seconds())
323+
e.error.Set(totalErrors)
329324
}
330325

331326
func (e *Exporter) connect() error {

‎go.sum

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
2+
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
13
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
24
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
35
github.com/UNO-SOFT/zlog v0.8.1 h1:TEFkGJHtUfTRgMkLZiAjLSHALjwSBdw6/zByMC5GJt4=
@@ -21,6 +23,7 @@ github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi
2123
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
2224
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
2325
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
26+
github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
2427
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
2528
github.com/godror/godror v0.44.6 h1:3Aw45jpKkPsss80DVy4/XiqXUPCsHmW0efSHBQFi0h4=
2629
github.com/godror/godror v0.44.6/go.mod h1:KJwMtQpK9o3WdEiNw7qvgSk827YDLj9MV/bXSzvUzlo=
@@ -40,10 +43,16 @@ github.com/godror/knownpb v0.1.2 h1:icMyYsYVpGmzhoVA01xyd0o4EaubR31JPK1UxQWe4kM=
4043
github.com/godror/knownpb v0.1.2/go.mod h1:zs9hH+lwj7mnPHPnKCcxdOGz38Axa9uT+97Ng+Nnu5s=
4144
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
4245
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
46+
github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
47+
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
4348
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
4449
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
4550
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
4651
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
52+
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
53+
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
54+
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
55+
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
4756
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
4857
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
4958
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -56,6 +65,10 @@ github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U
5665
github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
5766
github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ=
5867
github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE=
68+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
69+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
70+
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
71+
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
5972
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
6073
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
6174
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
@@ -121,6 +134,8 @@ golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
121134
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
122135
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
123136
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
137+
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
138+
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
124139
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
125140
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
126141
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
@@ -146,12 +161,17 @@ golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
146161
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
147162
golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
148163
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
164+
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
149165
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
150166
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
151167
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
152168
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
153169
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
154170
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
171+
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
172+
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
173+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
174+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
155175
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
156176
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
157177
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

0 commit comments

Comments
 (0)