1
0
Fork 0
mirror of https://github.com/TwiN/gatus.git synced 2024-12-15 17:51:09 +00:00

Prevent multiple services from being evaluated at the same time

This commit is contained in:
TwinProduction 2020-04-06 18:58:13 -04:00
parent ab73c4666e
commit fe82465c19
7 changed files with 36 additions and 32 deletions

View file

@ -18,7 +18,7 @@ You can specify a custom path by setting the `GATUS_CONFIG_FILE` environment var
metrics: true # Whether to expose metrics at /metrics metrics: true # Whether to expose metrics at /metrics
services: services:
- name: twinnation # Name of your service, can be anything - name: twinnation # Name of your service, can be anything
url: https://twinnation.org/actuator/health url: https://twinnation.org/health
interval: 15s # Duration to wait between every status check (opt. default: 10s) interval: 15s # Duration to wait between every status check (opt. default: 10s)
conditions: conditions:
- "[STATUS] == 200" - "[STATUS] == 200"

View file

@ -1,7 +1,7 @@
metrics: true metrics: true
services: services:
- name: Twinnation - name: Twinnation
url: https://twinnation.org/actuator/health url: https://twinnation.org/health
interval: 30s interval: 30s
conditions: conditions:
- "[STATUS] == 200" - "[STATUS] == 200"

View file

@ -76,7 +76,7 @@ func (service *Service) EvaluateConditions() *Result {
result.Success = false result.Success = false
} }
for _, condition := range service.Conditions { for _, condition := range service.Conditions {
success := condition.Evaluate(result) success := condition.evaluate(result)
if !success { if !success {
result.Success = false result.Success = false
} }
@ -93,7 +93,7 @@ type ConditionResult struct {
type Condition string type Condition string
func (c *Condition) Evaluate(result *Result) bool { func (c *Condition) evaluate(result *Result) bool {
condition := string(*c) condition := string(*c)
if strings.Contains(condition, "==") { if strings.Contains(condition, "==") {
parts := sanitizeAndResolve(strings.Split(condition, "=="), result) parts := sanitizeAndResolve(strings.Split(condition, "=="), result)

View file

@ -7,7 +7,7 @@ import (
func TestEvaluateWithIp(t *testing.T) { func TestEvaluateWithIp(t *testing.T) {
condition := Condition("[IP] == 127.0.0.1") condition := Condition("[IP] == 127.0.0.1")
result := &Result{Ip: "127.0.0.1"} result := &Result{Ip: "127.0.0.1"}
condition.Evaluate(result) condition.evaluate(result)
if !result.ConditionResults[0].Success { if !result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a success", condition) t.Errorf("Condition '%s' should have been a success", condition)
} }
@ -16,7 +16,7 @@ func TestEvaluateWithIp(t *testing.T) {
func TestEvaluateWithStatus(t *testing.T) { func TestEvaluateWithStatus(t *testing.T) {
condition := Condition("[STATUS] == 201") condition := Condition("[STATUS] == 201")
result := &Result{HttpStatus: 201} result := &Result{HttpStatus: 201}
condition.Evaluate(result) condition.evaluate(result)
if !result.ConditionResults[0].Success { if !result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a success", condition) t.Errorf("Condition '%s' should have been a success", condition)
} }
@ -25,7 +25,7 @@ func TestEvaluateWithStatus(t *testing.T) {
func TestEvaluateWithFailure(t *testing.T) { func TestEvaluateWithFailure(t *testing.T) {
condition := Condition("[STATUS] == 200") condition := Condition("[STATUS] == 200")
result := &Result{HttpStatus: 500} result := &Result{HttpStatus: 500}
condition.Evaluate(result) condition.evaluate(result)
if result.ConditionResults[0].Success { if result.ConditionResults[0].Success {
t.Errorf("Condition '%s' should have been a failure", condition) t.Errorf("Condition '%s' should have been a failure", condition)
} }

View file

@ -1,7 +1,7 @@
metrics: true metrics: true
services: services:
- name: TwiNNatioN - name: TwiNNatioN
url: https://twinnation.org/actuator/health url: https://twinnation.org/health
interval: 10s interval: 10s
conditions: conditions:
- "[STATUS] == 200" - "[STATUS] == 200"

View file

@ -4,7 +4,7 @@ data:
metrics: true metrics: true
services: services:
- name: TwiNNatioN - name: TwiNNatioN
url: https://twinnation.org/actuator/health url: https://twinnation.org/health
interval: 1m interval: 1m
conditions: conditions:
- "[STATUS] == 200" - "[STATUS] == 200"

View file

@ -20,28 +20,32 @@ func GetServiceResults() *map[string][]*core.Result {
func Monitor(cfg *config.Config) { func Monitor(cfg *config.Config) {
for _, service := range cfg.Services { for _, service := range cfg.Services {
go func(service *core.Service) { go monitor(service)
for { // To prevent multiple requests from running at the same time
log.Printf("[watchdog][Monitor] Monitoring serviceName=%s", service.Name) time.Sleep(500 * time.Millisecond)
result := service.EvaluateConditions() }
metric.PublishMetricsForService(service, result) }
rwLock.Lock()
serviceResults[service.Name] = append(serviceResults[service.Name], result) func monitor(service *core.Service) {
if len(serviceResults[service.Name]) > 20 { for {
serviceResults[service.Name] = serviceResults[service.Name][1:] // By placing the lock here, we prevent multiple services from being monitored at the exact same time, which
} // could cause performance issues and return inaccurate results
rwLock.Unlock() rwLock.Lock()
log.Printf( log.Printf("[watchdog][Monitor] Monitoring serviceName=%s", service.Name)
"[watchdog][Monitor] Finished monitoring serviceName=%s; errors=%d; requestDuration=%s", result := service.EvaluateConditions()
service.Name, metric.PublishMetricsForService(service, result)
len(result.Errors), serviceResults[service.Name] = append(serviceResults[service.Name], result)
result.Duration.Round(time.Millisecond), if len(serviceResults[service.Name]) > 20 {
) serviceResults[service.Name] = serviceResults[service.Name][1:]
log.Printf("[watchdog][Monitor] Waiting interval=%s before monitoring serviceName=%s", service.Interval, service.Name) }
time.Sleep(service.Interval) rwLock.Unlock()
} log.Printf(
}(service) "[watchdog][Monitor] Finished monitoring serviceName=%s; errors=%d; requestDuration=%s",
// To prevent multiple requests from running exactly at the same time service.Name,
time.Sleep(100 * time.Millisecond) len(result.Errors),
result.Duration.Round(time.Millisecond),
)
log.Printf("[watchdog][Monitor] Waiting interval=%s before monitoring serviceName=%s", service.Interval, service.Name)
time.Sleep(service.Interval)
} }
} }