mirrors/twin-gatus
1
0
Fork 0
mirror of https://github.com/TwiN/gatus.git synced 2024-12-14 11:58:04 +00:00
Code Activity

Start working on #12: Fix inconsistencies in alerting providers

Browse source
This commit is contained in:
TwinProduction 2020-09-19 16:22:12 -04:00
parent abd49bca8e
commit ae2c4b1ea9
16 changed files with 337 additions and 240 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
README.md
View file

@ -94,15 +94,17 @@ Note that you can also add environment variables in the configuration file (i.e.
| `services[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved | `false` |
| `services[].alerts[].description` | Description of the alert. Will be included in the alert sent | `""` |
| `alerting` | Configuration for alerting | `{}` |
| `alerting.slack` | Webhook to use for alerts of type `slack` | `""` |
| `alerting.pagerduty` | PagerDuty Events API v2 integration key. Used for alerts of type `pagerduty` | `""` |
| `alerting.slack` | Configuration for alerts of type `slack` | `""` |
| `alerting.slack.webhook-url` | Slack Webhook URL | Required `""` |
| `alerting.pagerduty` | Configuration for alerts of type `pagerduty` | `""` |
| `alerting.pagerduty.integration-key` | PagerDuty Events API v2 integration key. | Required `""` |
| `alerting.twilio` | Settings for alerts of type `twilio` | `""` |
| `alerting.twilio.sid` | Twilio account SID | Required `""` |
| `alerting.twilio.token` | Twilio auth token | Required `""` |
| `alerting.twilio.from` | Number to send Twilio alerts from | Required `""` |
| `alerting.twilio.to` | Number to send twilio alerts to | Required `""` |
| `alerting.custom` | Configuration for custom actions on failure or alerts | `""` |
| `alerting.custom.url` | Custom alerting request url | `""` |
| `alerting.custom.url` | Custom alerting request url | Required `""` |
| `alerting.custom.body` | Custom alerting request body. | `""` |
| `alerting.custom.headers` | Custom alerting request headers | `{}` |
@ -133,7 +135,8 @@ Here are some examples of conditions you can use:
```yaml
alerting:
slack: "https://hooks.slack.com/services/**********/**********/**********"
slack:
webhook-url: "https://hooks.slack.com/services/**********/**********/**********"
services:
- name: twinnation
interval: 30s
@ -168,7 +171,8 @@ PagerDuty instead.
```yaml
alerting:
pagerduty: "********************************"
pagerduty:
integration-key: "********************************"
services:
- name: twinnation
interval: 30s

8
alerting/config.go Normal file
View file

@ -0,0 +1,8 @@
package alerting
type Config struct {
Slack *SlackAlertProvider `yaml:"slack"`
PagerDuty *PagerDutyAlertProvider `yaml:"pagerduty"`
Twilio *TwilioAlertProvider `yaml:"twilio"`
Custom *CustomAlertProvider `yaml:"custom"`
}

80
alerting/custom.go Normal file
View file

@ -0,0 +1,80 @@
package alerting
import (
"bytes"
"fmt"
"github.com/TwinProduction/gatus/client"
"io/ioutil"
"net/http"
"strings"
)
type CustomAlertProvider struct {
Url string `yaml:"url"`
Method string `yaml:"method,omitempty"`
Body string `yaml:"body,omitempty"`
Headers map[string]string `yaml:"headers,omitempty"`
}
func (provider *CustomAlertProvider) IsValid() bool {
return len(provider.Url) > 0
}
func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request {
body := provider.Body
providerUrl := provider.Url
method := provider.Method
if strings.Contains(body, "[ALERT_DESCRIPTION]") {
body = strings.ReplaceAll(body, "[ALERT_DESCRIPTION]", alertDescription)
}
if strings.Contains(body, "[SERVICE_NAME]") {
body = strings.ReplaceAll(body, "[SERVICE_NAME]", serviceName)
}
if strings.Contains(body, "[ALERT_TRIGGERED_OR_RESOLVED]") {
if resolved {
body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
} else {
body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
}
}
if strings.Contains(providerUrl, "[ALERT_DESCRIPTION]") {
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_DESCRIPTION]", alertDescription)
}
if strings.Contains(providerUrl, "[SERVICE_NAME]") {
providerUrl = strings.ReplaceAll(providerUrl, "[SERVICE_NAME]", serviceName)
}
if strings.Contains(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]") {
if resolved {
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
} else {
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
}
}
if len(method) == 0 {
method = "GET"
}
bodyBuffer := bytes.NewBuffer([]byte(body))
request, _ := http.NewRequest(method, providerUrl, bodyBuffer)
for k, v := range provider.Headers {
request.Header.Set(k, v)
}
return request
}
// Send a request to the alert provider and return the body
func (provider *CustomAlertProvider) Send(serviceName, alertDescription string, resolved bool) ([]byte, error) {
request := provider.buildRequest(serviceName, alertDescription, resolved)
response, err := client.GetHttpClient().Do(request)
if err != nil {
return nil, err
}
if response.StatusCode > 399 {
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, fmt.Errorf("call to provider alert returned status code %d", response.StatusCode)
} else {
return nil, fmt.Errorf("call to provider alert returned status code %d: %s", response.StatusCode, string(body))
}
}
return ioutil.ReadAll(response.Body)
}

13
core/alerting_test.go → alerting/custom_test.go
View file

@ -1,10 +1,21 @@
package core
package alerting
import (
"io/ioutil"
"testing"
)
func TestCustomAlertProvider_IsValid(t *testing.T) {
invalidProvider := CustomAlertProvider{Url: ""}
if invalidProvider.IsValid() {
t.Error("provider shouldn't have been valid")
}
validProvider := CustomAlertProvider{Url: "http://example.com"}
if !validProvider.IsValid() {
t.Error("provider should've been valid")
}
}
func TestCustomAlertProvider_buildRequestWhenResolved(t *testing.T) {
const (
ExpectedUrl = "http://example.com/service-name"

36
alerting/pagerduty.go
View file

@ -1,7 +1,35 @@
package alerting
type pagerDutyResponse struct {
Status string `json:"status"`
Message string `json:"message"`
DedupKey string `json:"dedup_key"`
import (
"fmt"
"github.com/TwinProduction/gatus/core"
)
type PagerDutyAlertProvider struct {
IntegrationKey string `yaml:"integration-key"`
}
func (provider *PagerDutyAlertProvider) IsValid() bool {
return len(provider.IntegrationKey) == 32
}
// https://developer.pagerduty.com/docs/events-api-v2/trigger-events/
func (provider *PagerDutyAlertProvider) ToCustomAlertProvider(eventAction, resolveKey string, service *core.Service, message string) *CustomAlertProvider {
return &CustomAlertProvider{
Url: "https://events.pagerduty.com/v2/enqueue",
Method: "POST",
Body: fmt.Sprintf(`{
"routing_key": "%s",
"dedup_key": "%s",
"event_action": "%s",
"payload": {
"summary": "%s",
"source": "%s",
"severity": "critical"
}
}`, provider.IntegrationKey, resolveKey, eventAction, message, service.Name),
Headers: map[string]string{
"Content-Type": "application/json",
},
}
}

14
alerting/pagerduty_test.go Normal file
View file

@ -0,0 +1,14 @@
package alerting
import "testing"
func TestPagerDutyAlertProvider_IsValid(t *testing.T) {
invalidProvider := PagerDutyAlertProvider{IntegrationKey: ""}
if invalidProvider.IsValid() {
t.Error("provider shouldn't have been valid")
}
validProvider := PagerDutyAlertProvider{IntegrationKey: "00000000000000000000000000000000"}
if !validProvider.IsValid() {
t.Error("provider should've been valid")
}
}

59
alerting/slack.go Normal file
View file

@ -0,0 +1,59 @@
package alerting
import (
"fmt"
"github.com/TwinProduction/gatus/core"
)
type SlackAlertProvider struct {
WebhookUrl string `yaml:"webhook-url"`
}
func (provider *SlackAlertProvider) IsValid() bool {
return len(provider.WebhookUrl) > 0
}
func (provider *SlackAlertProvider) ToCustomAlertProvider(service *core.Service, alert *core.Alert, result *core.Result, resolved bool) *CustomAlertProvider {
var message string
var color string
if resolved {
message = fmt.Sprintf("An alert for *%s* has been resolved after passing successfully %d time(s) in a row", service.Name, alert.SuccessThreshold)
color = "#36A64F"
} else {
message = fmt.Sprintf("An alert for *%s* has been triggered due to having failed %d time(s) in a row", service.Name, alert.FailureThreshold)
color = "#DD0000"
}
var results string
for _, conditionResult := range result.ConditionResults {
var prefix string
if conditionResult.Success {
prefix = ":heavy_check_mark:"
} else {
prefix = ":x:"
}
results += fmt.Sprintf("%s - `%s`\n", prefix, conditionResult.Condition)
}
return &CustomAlertProvider{
Url: provider.WebhookUrl,
Method: "POST",
Body: fmt.Sprintf(`{
"text": "",
"attachments": [
{
"title": ":helmet_with_white_cross: Gatus",
"text": "%s:\n> %s",
"short": false,
"color": "%s",
"fields": [
{
"title": "Condition results",
"value": "%s",
"short": false
}
]
},
]
}`, message, alert.Description, color, results),
Headers: map[string]string{"Content-Type": "application/json"},
}
}

14
alerting/slack_test.go Normal file
View file

@ -0,0 +1,14 @@
package alerting
import "testing"
func TestSlackAlertProvider_IsValid(t *testing.T) {
invalidProvider := SlackAlertProvider{WebhookUrl: ""}
if invalidProvider.IsValid() {
t.Error("provider shouldn't have been valid")
}
validProvider := SlackAlertProvider{WebhookUrl: "http://example.com"}
if !validProvider.IsValid() {
t.Error("provider should've been valid")
}
}

34
alerting/twilio.go Normal file
View file

@ -0,0 +1,34 @@
package alerting
import (
"encoding/base64"
"fmt"
"net/url"
)
type TwilioAlertProvider struct {
SID string `yaml:"sid"`
Token string `yaml:"token"`
From string `yaml:"from"`
To string `yaml:"to"`
}
func (provider *TwilioAlertProvider) IsValid() bool {
return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0
}
func (provider *TwilioAlertProvider) ToCustomAlertProvider(message string) *CustomAlertProvider {
return &CustomAlertProvider{
Url: fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID),
Method: "POST",
Body: url.Values{
"To": {provider.To},
"From": {provider.From},
"Body": {message},
}.Encode(),
Headers: map[string]string{
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", provider.SID, provider.Token)))),
},
}
}

19
alerting/twilio_test.go Normal file
View file

@ -0,0 +1,19 @@
package alerting
import "testing"
func TestTwilioAlertProvider_IsValid(t *testing.T) {
invalidProvider := TwilioAlertProvider{}
if invalidProvider.IsValid() {
t.Error("provider shouldn't have been valid")
}
validProvider := TwilioAlertProvider{
SID: "1",
Token: "1",
From: "1",
To: "1",
}
if !validProvider.IsValid() {
t.Error("provider should've been valid")
}
}

9
config/config.go
View file

@ -2,6 +2,7 @@ package config
import (
"errors"
"github.com/TwinProduction/gatus/alerting"
"github.com/TwinProduction/gatus/core"
"gopkg.in/yaml.v2"
"io/ioutil"
@ -21,10 +22,10 @@ var (
)
type Config struct {
Metrics bool `yaml:"metrics"`
Debug bool `yaml:"debug"`
Alerting *core.AlertingConfig `yaml:"alerting"`
Services []*core.Service `yaml:"services"`
Metrics bool `yaml:"metrics"`
Debug bool `yaml:"debug"`
Alerting *alerting.Config `yaml:"alerting"`
Services []*core.Service `yaml:"services"`
}
func Get() *Config {

5
config/config_test.go
View file

@ -121,7 +121,8 @@ badconfig:
func TestParseAndValidateConfigBytesWithAlerting(t *testing.T) {
config, err := parseAndValidateConfigBytes([]byte(`
alerting:
slack: "http://example.com"
slack:
webhook-url: "http://example.com"
services:
- name: twinnation
url: https://twinnation.org/actuator/health
@ -145,7 +146,7 @@ services:
if config.Alerting == nil {
t.Fatal("config.AlertingConfig shouldn't have been nil")
}
if config.Alerting.Slack != "http://example.com" {
if config.Alerting.Slack.WebhookUrl != "http://example.com" {
t.Errorf("Slack webhook should've been %s, but was %s", "http://example.com", config.Alerting.Slack)
}
if len(config.Services) != 1 {

178
core/alerting.go
View file

@ -1,178 +0,0 @@
package core
import (
"bytes"
"encoding/base64"
"fmt"
"github.com/TwinProduction/gatus/client"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
type AlertingConfig struct {
Slack string `yaml:"slack"`
PagerDuty string `yaml:"pagerduty"`
Twilio *TwilioAlertProvider `yaml:"twilio"`
Custom *CustomAlertProvider `yaml:"custom"`
}
type TwilioAlertProvider struct {
SID string `yaml:"sid"`
Token string `yaml:"token"`
From string `yaml:"from"`
To string `yaml:"to"`
}
func (provider *TwilioAlertProvider) IsValid() bool {
return len(provider.Token) > 0 && len(provider.SID) > 0 && len(provider.From) > 0 && len(provider.To) > 0
}
type CustomAlertProvider struct {
Url string `yaml:"url"`
Method string `yaml:"method,omitempty"`
Body string `yaml:"body,omitempty"`
Headers map[string]string `yaml:"headers,omitempty"`
}
func (provider *CustomAlertProvider) IsValid() bool {
return len(provider.Url) > 0
}
func (provider *CustomAlertProvider) buildRequest(serviceName, alertDescription string, resolved bool) *http.Request {
body := provider.Body
providerUrl := provider.Url
if strings.Contains(body, "[ALERT_DESCRIPTION]") {
body = strings.ReplaceAll(body, "[ALERT_DESCRIPTION]", alertDescription)
}
if strings.Contains(body, "[SERVICE_NAME]") {
body = strings.ReplaceAll(body, "[SERVICE_NAME]", serviceName)
}
if strings.Contains(body, "[ALERT_TRIGGERED_OR_RESOLVED]") {
if resolved {
body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
} else {
body = strings.ReplaceAll(body, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
}
}
if strings.Contains(providerUrl, "[ALERT_DESCRIPTION]") {
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_DESCRIPTION]", alertDescription)
}
if strings.Contains(providerUrl, "[SERVICE_NAME]") {
providerUrl = strings.ReplaceAll(providerUrl, "[SERVICE_NAME]", serviceName)
}
if strings.Contains(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]") {
if resolved {
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "RESOLVED")
} else {
providerUrl = strings.ReplaceAll(providerUrl, "[ALERT_TRIGGERED_OR_RESOLVED]", "TRIGGERED")
}
}
bodyBuffer := bytes.NewBuffer([]byte(body))
request, _ := http.NewRequest(provider.Method, providerUrl, bodyBuffer)
for k, v := range provider.Headers {
request.Header.Set(k, v)
}
return request
}
// Send a request to the alert provider and return the body
func (provider *CustomAlertProvider) Send(serviceName, alertDescription string, resolved bool) ([]byte, error) {
request := provider.buildRequest(serviceName, alertDescription, resolved)
response, err := client.GetHttpClient().Do(request)
if err != nil {
return nil, err
}
if response.StatusCode > 399 {
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, fmt.Errorf("call to provider alert returned status code %d", response.StatusCode)
} else {
return nil, fmt.Errorf("call to provider alert returned status code %d: %s", response.StatusCode, string(body))
}
}
return ioutil.ReadAll(response.Body)
}
func CreateSlackCustomAlertProvider(slackWebHookUrl string, service *Service, alert *Alert, result *Result, resolved bool) *CustomAlertProvider {
var message string
var color string
if resolved {
message = fmt.Sprintf("An alert for *%s* has been resolved after passing successfully %d time(s) in a row", service.Name, alert.SuccessThreshold)
color = "#36A64F"
} else {
message = fmt.Sprintf("An alert for *%s* has been triggered due to having failed %d time(s) in a row", service.Name, alert.FailureThreshold)
color = "#DD0000"
}
var results string
for _, conditionResult := range result.ConditionResults {
var prefix string
if conditionResult.Success {
prefix = ":heavy_check_mark:"
} else {
prefix = ":x:"
}
results += fmt.Sprintf("%s - `%s`\n", prefix, conditionResult.Condition)
}
return &CustomAlertProvider{
Url: slackWebHookUrl,
Method: "POST",
Body: fmt.Sprintf(`{
"text": "",
"attachments": [
{
"title": ":helmet_with_white_cross: Gatus",
"text": "%s:\n> %s",
"short": false,
"color": "%s",
"fields": [
{
"title": "Condition results",
"value": "%s",
"short": false
}
]
},
]
}`, message, alert.Description, color, results),
Headers: map[string]string{"Content-Type": "application/json"},
}
}
func CreateTwilioCustomAlertProvider(provider *TwilioAlertProvider, message string) *CustomAlertProvider {
return &CustomAlertProvider{
Url: fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID),
Method: "POST",
Body: url.Values{
"To": {provider.To},
"From": {provider.From},
"Body": {message},
}.Encode(),
Headers: map[string]string{
"Content-Type": "application/x-www-form-urlencoded",
"Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", provider.SID, provider.Token)))),
},
}
}
// https://developer.pagerduty.com/docs/events-api-v2/trigger-events/
func CreatePagerDutyCustomAlertProvider(routingKey, eventAction, resolveKey string, service *Service, message string) *CustomAlertProvider {
return &CustomAlertProvider{
Url: "https://events.pagerduty.com/v2/enqueue",
Method: "POST",
Body: fmt.Sprintf(`{
"routing_key": "%s",
"dedup_key": "%s",
"event_action": "%s",
"payload": {
"summary": "%s",
"source": "%s",
"severity": "critical"
}
}`, routingKey, resolveKey, eventAction, message, service.Name),
Headers: map[string]string{
"Content-Type": "application/json",
},
}
}

5
docs/pagerduty-integration-guide.md
View file

@ -30,10 +30,11 @@ If you need help with this integration, please create an issue at https://github
## In Gatus
In your configuration file, you must first specify the integration key in `alerting.pagerduty`, like so:
In your configuration file, you must first specify the integration key at `alerting.pagerduty.integration-key`, like so:
```yaml
alerting:
pagerduty: "********************************"
pagerduty:
integration-key: "********************************"
```
You can now add alerts of type `pagerduty` in the services you've defined, like so:
```yaml

86
alerting/alerting.go → watchdog/alerting.go
View file

@ -1,15 +1,16 @@
package alerting
package watchdog
import (
"encoding/json"
"fmt"
"github.com/TwinProduction/gatus/alerting"
"github.com/TwinProduction/gatus/config"
"github.com/TwinProduction/gatus/core"
"log"
)
// Handle takes care of alerts to resolve and alerts to trigger based on result success or failure
func Handle(service *core.Service, result *core.Result) {
// HandleAlerting takes care of alerts to resolve and alerts to trigger based on result success or failure
func HandleAlerting(service *core.Service, result *core.Result) {
cfg := config.Get()
if cfg.Alerting == nil {
return
@ -31,43 +32,38 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf
}
if alert.Triggered {
if cfg.Debug {
log.Printf("[alerting][handleAlertsToTrigger] Alert with description='%s' has already been triggered, skipping", alert.Description)
log.Printf("[watchdog][handleAlertsToTrigger] Alert with description='%s' has already been triggered, skipping", alert.Description)
}
continue
}
var alertProvider *core.CustomAlertProvider
var alertProvider *alerting.CustomAlertProvider
if alert.Type == core.SlackAlert {
if len(cfg.Alerting.Slack) > 0 {
log.Printf("[alerting][handleAlertsToTrigger] Sending Slack alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, false)
if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending Slack alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, false)
} else {
log.Printf("[alerting][handleAlertsToTrigger] Not sending Slack alert despite being triggered, because there is no Slack webhook configured")
log.Printf("[watchdog][handleAlertsToTrigger] Not sending Slack alert despite being triggered, because there is no Slack webhook configured")
}
} else if alert.Type == core.PagerDutyAlert {
if len(cfg.Alerting.PagerDuty) > 0 {
log.Printf("[alerting][handleAlertsToTrigger] Sending PagerDuty alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = core.CreatePagerDutyCustomAlertProvider(cfg.Alerting.PagerDuty, "trigger", "", service, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() {
log.Printf("[watchdog][handleAlertsToTrigger] Sending PagerDuty alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("trigger", "", service, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[alerting][handleAlertsToTrigger] Not sending PagerDuty alert despite being triggered, because PagerDuty isn't configured properly")
log.Printf("[watchdog][handleAlertsToTrigger] Not sending PagerDuty alert despite being triggered, because PagerDuty isn't configured properly")
}
} else if alert.Type == core.TwilioAlert {
if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
log.Printf("[alerting][handleAlertsToTrigger] Sending Twilio alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
log.Printf("[watchdog][handleAlertsToTrigger] Sending Twilio alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("TRIGGERED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[alerting][handleAlertsToTrigger] Not sending Twilio alert despite being triggered, because Twilio config settings missing")
log.Printf("[watchdog][handleAlertsToTrigger] Not sending Twilio alert despite being triggered, because Twilio config settings missing")
}
} else if alert.Type == core.CustomAlert {
if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
log.Printf("[alerting][handleAlertsToTrigger] Sending custom alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = &core.CustomAlertProvider{
Url: cfg.Alerting.Custom.Url,
Method: cfg.Alerting.Custom.Method,
Body: cfg.Alerting.Custom.Body,
Headers: cfg.Alerting.Custom.Headers,
}
log.Printf("[watchdog][handleAlertsToTrigger] Sending custom alert because alert with description='%s' has been triggered", alert.Description)
alertProvider = cfg.Alerting.Custom
} else {
log.Printf("[alerting][handleAlertsToTrigger] Not sending custom alert despite being triggered, because there is no custom url configured")
log.Printf("[watchdog][handleAlertsToTrigger] Not sending custom alert despite being triggered, because there is no custom url configured")
}
}
if alertProvider != nil {
@ -80,7 +76,7 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf
var response pagerDutyResponse
err = json.Unmarshal(body, &response)
if err != nil {
log.Printf("[alerting][handleAlertsToTrigger] Ran into error unmarshaling pager duty response: %s", err.Error())
log.Printf("[watchdog][handleAlertsToTrigger] Ran into error unmarshaling pager duty response: %s", err.Error())
} else {
alert.ResolveKey = response.DedupKey
}
@ -89,7 +85,7 @@ func handleAlertsToTrigger(service *core.Service, result *core.Result, cfg *conf
_, err = alertProvider.Send(service.Name, alert.Description, false)
}
if err != nil {
log.Printf("[alerting][handleAlertsToTrigger] Ran into error sending an alert: %s", err.Error())
log.Printf("[watchdog][handleAlertsToTrigger] Ran into error sending an alert: %s", err.Error())
} else {
alert.Triggered = true
}
@ -107,46 +103,46 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf
if !alert.SendOnResolved {
continue
}
var alertProvider *core.CustomAlertProvider
var alertProvider *alerting.CustomAlertProvider
if alert.Type == core.SlackAlert {
if len(cfg.Alerting.Slack) > 0 {
log.Printf("[alerting][handleAlertsToResolve] Sending Slack alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, true)
if cfg.Alerting.Slack != nil && cfg.Alerting.Slack.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending Slack alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = cfg.Alerting.Slack.ToCustomAlertProvider(service, alert, result, true)
} else {
log.Printf("[alerting][handleAlertsToResolve] Not sending Slack alert despite being resolved, because there is no Slack webhook configured")
log.Printf("[watchdog][handleAlertsToResolve] Not sending Slack alert despite being resolved, because there is no Slack webhook configured")
}
} else if alert.Type == core.PagerDutyAlert {
if len(cfg.Alerting.PagerDuty) > 0 {
log.Printf("[alerting][handleAlertsToResolve] Sending PagerDuty alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = core.CreatePagerDutyCustomAlertProvider(cfg.Alerting.PagerDuty, "resolve", alert.ResolveKey, service, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
if cfg.Alerting.PagerDuty != nil && cfg.Alerting.PagerDuty.IsValid() {
log.Printf("[watchdog][handleAlertsToResolve] Sending PagerDuty alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = cfg.Alerting.PagerDuty.ToCustomAlertProvider("resolve", alert.ResolveKey, service, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[alerting][handleAlertsToResolve] Not sending PagerDuty alert despite being resolved, because PagerDuty isn't configured properly")
log.Printf("[watchdog][handleAlertsToResolve] Not sending PagerDuty alert despite being resolved, because PagerDuty isn't configured properly")
}
} else if alert.Type == core.TwilioAlert {
if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() {
log.Printf("[alerting][handleAlertsToResolve] Sending Twilio alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
log.Printf("[watchdog][handleAlertsToResolve] Sending Twilio alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = cfg.Alerting.Twilio.ToCustomAlertProvider(fmt.Sprintf("RESOLVED: %s - %s", service.Name, alert.Description))
} else {
log.Printf("[alerting][handleAlertsToResolve] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly")
log.Printf("[watchdog][handleAlertsToResolve] Not sending Twilio alert despite being resolved, because Twilio isn't configured properly")
}
} else if alert.Type == core.CustomAlert {
if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {
log.Printf("[alerting][handleAlertsToResolve] Sending custom alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = &core.CustomAlertProvider{
log.Printf("[watchdog][handleAlertsToResolve] Sending custom alert because alert with description='%s' has been resolved", alert.Description)
alertProvider = &alerting.CustomAlertProvider{
Url: cfg.Alerting.Custom.Url,
Method: cfg.Alerting.Custom.Method,
Body: cfg.Alerting.Custom.Body,
Headers: cfg.Alerting.Custom.Headers,
}
} else {
log.Printf("[alerting][handleAlertsToResolve] Not sending custom alert despite being resolved, because the custom provider isn't configured properly")
log.Printf("[watchdog][handleAlertsToResolve] Not sending custom alert despite being resolved, because the custom provider isn't configured properly")
}
}
if alertProvider != nil {
// TODO: retry on error
_, err := alertProvider.Send(service.Name, alert.Description, true)
if err != nil {
log.Printf("[alerting][handleAlertsToResolve] Ran into error sending an alert: %s", err.Error())
log.Printf("[watchdog][handleAlertsToResolve] Ran into error sending an alert: %s", err.Error())
} else {
if alert.Type == core.PagerDutyAlert {
alert.ResolveKey = ""
@ -156,3 +152,9 @@ func handleAlertsToResolve(service *core.Service, result *core.Result, cfg *conf
}
service.NumberOfFailuresInARow = 0
}
type pagerDutyResponse struct {
Status string `json:"status"`
Message string `json:"message"`
DedupKey string `json:"dedup_key"`
}

3
watchdog/watchdog.go
View file

@ -3,7 +3,6 @@ package watchdog
import (
"encoding/json"
"fmt"
"github.com/TwinProduction/gatus/alerting"
"github.com/TwinProduction/gatus/config"
"github.com/TwinProduction/gatus/core"
"github.com/TwinProduction/gatus/metric"
@ -71,7 +70,7 @@ func monitor(service *core.Service) {
result.Duration.Round(time.Millisecond),
extra,
)
alerting.Handle(service, result)
HandleAlerting(service, result)
if cfg.Debug {
log.Printf("[watchdog][monitor] Waiting for interval=%s before monitoring serviceName=%s again", service.Interval, service.Name)
}