mirrors/twin-gatus
1
0
Fork 0
mirror of https://github.com/TwiN/gatus.git synced 2024-12-14 11:58:04 +00:00
Code Activity

fix(web): Allow configuration of read-buffer-size (#675)

Browse source 
This fixes the `431 Request Header Fields Too Large` error

By default, the read-buffer-size is 8192, up from fiber's default of 4096.

Fixes #674

Fixes #636

Supersedes #637

Supersedes #663
This commit is contained in:
TwiN 2024-02-07 18:54:30 -05:00 committed by GitHub
parent 3d1b4e566d
commit 2a623a59d3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 157 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
README.md
View file

@ -206,57 +206,58 @@ If you want to test it locally, see [Docker](#docker).
## Configuration ## Configuration
| Parameter | Description | Default | | Parameter | Description | Default |
|:------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------| |:------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------|
| `debug` | Whether to enable debug logs. | `false` | | `debug` | Whether to enable debug logs. | `false` |
| `metrics` | Whether to expose metrics at /metrics. | `false` | | `metrics` | Whether to expose metrics at /metrics. | `false` |
| `storage` | [Storage configuration](#storage) | `{}` | | `storage` | [Storage configuration](#storage) | `{}` |
| `endpoints` | List of endpoints to monitor. | Required `[]` | | `endpoints` | List of endpoints to monitor. | Required `[]` |
| `endpoints[].enabled` | Whether to monitor the endpoint. | `true` | | `endpoints[].enabled` | Whether to monitor the endpoint. | `true` |
| `endpoints[].name` | Name of the endpoint. Can be anything. | Required `""` | | `endpoints[].name` | Name of the endpoint. Can be anything. | Required `""` |
| `endpoints[].group` | Group name. Used to group multiple endpoints together on the dashboard. <br />See [Endpoint groups](#endpoint-groups). | `""` | | `endpoints[].group` | Group name. Used to group multiple endpoints together on the dashboard. <br />See [Endpoint groups](#endpoint-groups). | `""` |
| `endpoints[].url` | URL to send the request to. | Required `""` | | `endpoints[].url` | URL to send the request to. | Required `""` |
| `endpoints[].method` | Request method. | `GET` | | `endpoints[].method` | Request method. | `GET` |
| `endpoints[].conditions` | Conditions used to determine the health of the endpoint. <br />See [Conditions](#conditions). | `[]` | | `endpoints[].conditions` | Conditions used to determine the health of the endpoint. <br />See [Conditions](#conditions). | `[]` |
| `endpoints[].interval` | Duration to wait between every status check. | `60s` | | `endpoints[].interval` | Duration to wait between every status check. | `60s` |
| `endpoints[].graphql` | Whether to wrap the body in a query param (`{"query":"$body"}`). | `false` | | `endpoints[].graphql` | Whether to wrap the body in a query param (`{"query":"$body"}`). | `false` |
| `endpoints[].body` | Request body. | `""` | | `endpoints[].body` | Request body. | `""` |
| `endpoints[].headers` | Request headers. | `{}` | | `endpoints[].headers` | Request headers. | `{}` |
| `endpoints[].dns` | Configuration for an endpoint of type DNS. <br />See [Monitoring an endpoint using DNS queries](#monitoring-an-endpoint-using-dns-queries). | `""` | | `endpoints[].dns` | Configuration for an endpoint of type DNS. <br />See [Monitoring an endpoint using DNS queries](#monitoring-an-endpoint-using-dns-queries). | `""` |
| `endpoints[].dns.query-type` | Query type (e.g. MX) | `""` | | `endpoints[].dns.query-type` | Query type (e.g. MX) | `""` |
| `endpoints[].dns.query-name` | Query name (e.g. example.com) | `""` | | `endpoints[].dns.query-name` | Query name (e.g. example.com) | `""` |
| `endpoints[].ssh` | Configuration for an endpoint of type SSH. <br />See [Monitoring an endpoint using SSH](#monitoring-an-endpoint-using-ssh). | `""` | | `endpoints[].ssh` | Configuration for an endpoint of type SSH. <br />See [Monitoring an endpoint using SSH](#monitoring-an-endpoint-using-ssh). | `""` |
| `endpoints[].ssh.username` | SSH username (e.g. example) | Required `""` | | `endpoints[].ssh.username` | SSH username (e.g. example) | Required `""` |
| `endpoints[].ssh.password` | SSH password (e.g. password) | Required `""` | | `endpoints[].ssh.password` | SSH password (e.g. password) | Required `""` |
| `endpoints[].alerts[].type` | Type of alert. <br />See [Alerting](#alerting) for all valid types. | Required `""` | | `endpoints[].alerts[].type` | Type of alert. <br />See [Alerting](#alerting) for all valid types. | Required `""` |
| `endpoints[].alerts[].enabled` | Whether to enable the alert. | `true` | | `endpoints[].alerts[].enabled` | Whether to enable the alert. | `true` |
| `endpoints[].alerts[].failure-threshold` | Number of failures in a row needed before triggering the alert. | `3` | | `endpoints[].alerts[].failure-threshold` | Number of failures in a row needed before triggering the alert. | `3` |
| `endpoints[].alerts[].success-threshold` | Number of successes in a row before an ongoing incident is marked as resolved. | `2` | | `endpoints[].alerts[].success-threshold` | Number of successes in a row before an ongoing incident is marked as resolved. | `2` |
| `endpoints[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved. | `false` | | `endpoints[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert is marked as resolved. | `false` |
| `endpoints[].alerts[].description` | Description of the alert. Will be included in the alert sent. | `""` | | `endpoints[].alerts[].description` | Description of the alert. Will be included in the alert sent. | `""` |
| `endpoints[].client` | [Client configuration](#client-configuration). | `{}` | | `endpoints[].client` | [Client configuration](#client-configuration). | `{}` |
| `endpoints[].ui` | UI configuration at the endpoint level. | `{}` | | `endpoints[].ui` | UI configuration at the endpoint level. | `{}` |
| `endpoints[].ui.hide-hostname` | Whether to hide the hostname in the result. | `false` | | `endpoints[].ui.hide-hostname` | Whether to hide the hostname in the result. | `false` |
| `endpoints[].ui.hide-url` | Whether to ensure the URL is not displayed in the results. Useful if the URL contains a token. | `false` | | `endpoints[].ui.hide-url` | Whether to ensure the URL is not displayed in the results. Useful if the URL contains a token. | `false` |
| `endpoints[].ui.dont-resolve-failed-conditions` | Whether to resolve failed conditions for the UI. | `false` | | `endpoints[].ui.dont-resolve-failed-conditions` | Whether to resolve failed conditions for the UI. | `false` |
| `endpoints[].ui.badge.reponse-time` | List of response time thresholds. Each time a threshold is reached, the badge has a different color. | `[50, 200, 300, 500, 750]` | | `endpoints[].ui.badge.reponse-time` | List of response time thresholds. Each time a threshold is reached, the badge has a different color. | `[50, 200, 300, 500, 750]` |
| `alerting` | [Alerting configuration](#alerting). | `{}` | | `alerting` | [Alerting configuration](#alerting). | `{}` |
| `security` | [Security configuration](#security). | `{}` | | `security` | [Security configuration](#security). | `{}` |
| `disable-monitoring-lock` | Whether to [disable the monitoring lock](#disable-monitoring-lock). | `false` | | `disable-monitoring-lock` | Whether to [disable the monitoring lock](#disable-monitoring-lock). | `false` |
| `skip-invalid-config-update` | Whether to ignore invalid configuration update. <br />See [Reloading configuration on the fly](#reloading-configuration-on-the-fly). | `false` | | `skip-invalid-config-update` | Whether to ignore invalid configuration update. <br />See [Reloading configuration on the fly](#reloading-configuration-on-the-fly). | `false` |
| `web` | Web configuration. | `{}` | | `web` | Web configuration. | `{}` |
| `web.address` | Address to listen on. | `0.0.0.0` | | `web.address` | Address to listen on. | `0.0.0.0` |
| `web.port` | Port to listen on. | `8080` | | `web.port` | Port to listen on. | `8080` |
| `web.tls.certificate-file` | Optional public certificate file for TLS in PEM format. | `` | | `web.read-buffer-size` | Buffer size for reading requests from a connection. Also limit for the maximum header size. | `8192` |
| `web.tls.private-key-file` | Optional private key file for TLS in PEM format. | `` | | `web.tls.certificate-file` | Optional public certificate file for TLS in PEM format. | `` |
| `ui` | UI configuration. | `{}` | | `web.tls.private-key-file` | Optional private key file for TLS in PEM format. | `` |
| `ui.title` | [Title of the document](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/title). | `Health Dashboard ǀ Gatus` | | `ui` | UI configuration. | `{}` |
| `ui.description` | Meta description for the page. | `Gatus is an advanced...`. | | `ui.title` | [Title of the document](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/title). | `Health Dashboard ǀ Gatus` |
| `ui.header` | Header at the top of the dashboard. | `Health Status` | | `ui.description` | Meta description for the page. | `Gatus is an advanced...`. |
| `ui.logo` | URL to the logo to display. | `""` | | `ui.header` | Header at the top of the dashboard. | `Health Status` |
| `ui.link` | Link to open when the logo is clicked. | `""` | | `ui.logo` | URL to the logo to display. | `""` |
| `ui.buttons` | List of buttons to display below the header. | `[]` | | `ui.link` | Link to open when the logo is clicked. | `""` |
| `ui.buttons[].name` | Text to display on the button. | Required `""` | | `ui.buttons` | List of buttons to display below the header. | `[]` |
| `ui.buttons[].link` | Link to open when the button is clicked. | Required `""` | | `ui.buttons[].name` | Text to display on the button. | Required `""` |
| `maintenance` | [Maintenance configuration](#maintenance). | `{}` | | `ui.buttons[].link` | Link to open when the button is clicked. | Required `""` |
| `maintenance` | [Maintenance configuration](#maintenance). | `{}` |
### Conditions ### Conditions
@ -1909,6 +1910,17 @@ endpoints:
</details> </details>
### How to fix 431 Request Header Fields Too Large error
Depending on where your environment is deployed and what kind of middleware or reverse proxy sits in front of Gatus,
you may run into this issue. This could be because the request headers are too large, e.g. big cookies.
By default, `web.read-buffer-size` is set to `8192`, but increasing this value like so will increase the read buffer size:
```yaml
web:
read-buffer-size: 32768
```
### Badges ### Badges
#### Uptime #### Uptime
![Uptime 1h](https://status.twin.sh/api/v1/endpoints/core_blog-external/uptimes/1h/badge.svg) ![Uptime 1h](https://status.twin.sh/api/v1/endpoints/core_blog-external/uptimes/1h/badge.svg)

8
api/api.go
View file

@ -7,6 +7,7 @@ import (
"os" "os"
"github.com/TwiN/gatus/v5/config" "github.com/TwiN/gatus/v5/config"
"github.com/TwiN/gatus/v5/config/web"
static "github.com/TwiN/gatus/v5/web" static "github.com/TwiN/gatus/v5/web"
"github.com/TwiN/health" "github.com/TwiN/health"
fiber "github.com/gofiber/fiber/v2" fiber "github.com/gofiber/fiber/v2"
@ -26,6 +27,10 @@ type API struct {
func New(cfg *config.Config) *API { func New(cfg *config.Config) *API {
api := &API{} api := &API{}
if cfg.Web == nil {
log.Println("[api.New] nil web config passed as parameter. This should only happen in tests. Using default web configuration")
cfg.Web = web.GetDefaultConfig()
}
api.router = api.createRouter(cfg) api.router = api.createRouter(cfg)
return api return api
} }
@ -40,7 +45,8 @@ func (a *API) createRouter(cfg *config.Config) *fiber.App {
log.Printf("[api.ErrorHandler] %s", err.Error()) log.Printf("[api.ErrorHandler] %s", err.Error())
return fiber.DefaultErrorHandler(c, err) return fiber.DefaultErrorHandler(c, err)
}, },
Network: fiber.NetworkTCP, ReadBufferSize: cfg.Web.ReadBufferSize,
Network: fiber.NetworkTCP,
}) })
if os.Getenv("ENVIRONMENT") == "dev" { if os.Getenv("ENVIRONMENT") == "dev" {
app.Use(cors.New(cors.Config{ app.Use(cors.New(cors.Config{

30
config/web/web.go
View file

@ -13,10 +13,16 @@ const (
// DefaultPort is the default port the application will listen on // DefaultPort is the default port the application will listen on
DefaultPort = 8080 DefaultPort = 8080
// DefaultReadBufferSize is the default value for ReadBufferSize
DefaultReadBufferSize = 8192
// MinimumReadBufferSize is the minimum value for ReadBufferSize, and also the default value set
// for fiber.Config.ReadBufferSize
MinimumReadBufferSize = 4096
) )
// Config is the structure which supports the configuration of the endpoint // Config is the structure which supports the configuration of the server listening to requests
// which provides access to the web frontend
type Config struct { type Config struct {
// Address to listen on (defaults to 0.0.0.0 specified by DefaultAddress) // Address to listen on (defaults to 0.0.0.0 specified by DefaultAddress)
Address string `yaml:"address"` Address string `yaml:"address"`
@ -24,6 +30,14 @@ type Config struct {
// Port to listen on (default to 8080 specified by DefaultPort) // Port to listen on (default to 8080 specified by DefaultPort)
Port int `yaml:"port"` Port int `yaml:"port"`
// ReadBufferSize sets fiber.Config.ReadBufferSize, which is the buffer size for reading requests coming from a
// single connection and also acts as a limit for the maximum header size.
//
// If you're getting occasional "Request Header Fields Too Large", you may want to try increasing this value.
//
// Defaults to DefaultReadBufferSize
ReadBufferSize int `yaml:"read-buffer-size,omitempty"`
// TLS configuration (optional) // TLS configuration (optional)
TLS *TLSConfig `yaml:"tls,omitempty"` TLS *TLSConfig `yaml:"tls,omitempty"`
} }
@ -38,7 +52,11 @@ type TLSConfig struct {
// GetDefaultConfig returns a Config struct with the default values // GetDefaultConfig returns a Config struct with the default values
func GetDefaultConfig() *Config { func GetDefaultConfig() *Config {
return &Config{Address: DefaultAddress, Port: DefaultPort} return &Config{
Address: DefaultAddress,
Port: DefaultPort,
ReadBufferSize: DefaultReadBufferSize,
}
} }
// ValidateAndSetDefaults validates the web configuration and sets the default values if necessary. // ValidateAndSetDefaults validates the web configuration and sets the default values if necessary.
@ -53,6 +71,12 @@ func (web *Config) ValidateAndSetDefaults() error {
} else if web.Port < 0 || web.Port > math.MaxUint16 { } else if web.Port < 0 || web.Port > math.MaxUint16 {
return fmt.Errorf("invalid port: value should be between %d and %d", 0, math.MaxUint16) return fmt.Errorf("invalid port: value should be between %d and %d", 0, math.MaxUint16)
} }
// Validate ReadBufferSize
if web.ReadBufferSize == 0 {
web.ReadBufferSize = DefaultReadBufferSize // Not set? Use the default value.
} else if web.ReadBufferSize < MinimumReadBufferSize {
web.ReadBufferSize = MinimumReadBufferSize // Below the minimum? Use the minimum value.
}
// Try to load the TLS certificates // Try to load the TLS certificates
if web.TLS != nil { if web.TLS != nil {
if err := web.TLS.isValid(); err != nil { if err := web.TLS.isValid(); err != nil {

89
config/web/web_test.go
View file

@ -12,6 +12,9 @@ func TestGetDefaultConfig(t *testing.T) {
if defaultConfig.Address != DefaultAddress { if defaultConfig.Address != DefaultAddress {
t.Error("expected default config to have the default address") t.Error("expected default config to have the default address")
} }
if defaultConfig.ReadBufferSize != DefaultReadBufferSize {
t.Error("expected default config to have the default read buffer size")
}
if defaultConfig.TLS != nil { if defaultConfig.TLS != nil {
t.Error("expected default config to have TLS disabled") t.Error("expected default config to have TLS disabled")
} }
@ -19,18 +22,20 @@ func TestGetDefaultConfig(t *testing.T) {
func TestConfig_ValidateAndSetDefaults(t *testing.T) { func TestConfig_ValidateAndSetDefaults(t *testing.T) {
scenarios := []struct { scenarios := []struct {
name string name string
cfg *Config cfg *Config
expectedAddress string expectedAddress string
expectedPort int expectedPort int
expectedErr bool expectedReadBufferSize int
expectedErr bool
}{ }{
{ {
name: "no-explicit-config", name: "no-explicit-config",
cfg: &Config{}, cfg: &Config{},
expectedAddress: "0.0.0.0", expectedAddress: "0.0.0.0",
expectedPort: 8080, expectedPort: 8080,
expectedErr: false, expectedReadBufferSize: 8192,
expectedErr: false,
}, },
{ {
name: "invalid-port", name: "invalid-port",
@ -38,25 +43,52 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) {
expectedErr: true, expectedErr: true,
}, },
{ {
name: "with-good-tls-config", name: "read-buffer-size-below-minimum",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/cert.pem", PrivateKeyFile: "../../testdata/cert.key"}}, cfg: &Config{ReadBufferSize: 1024},
expectedAddress: "0.0.0.0", expectedAddress: "0.0.0.0",
expectedPort: 443, expectedPort: 8080,
expectedErr: false, expectedReadBufferSize: MinimumReadBufferSize, // minimum is 4096, default is 8192.
expectedErr: false,
}, },
{ {
name: "with-bad-tls-config", name: "read-buffer-size-at-minimum",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/badcert.pem", PrivateKeyFile: "../../testdata/cert.key"}}, cfg: &Config{ReadBufferSize: MinimumReadBufferSize},
expectedAddress: "0.0.0.0", expectedAddress: "0.0.0.0",
expectedPort: 443, expectedPort: 8080,
expectedErr: true, expectedReadBufferSize: 4096,
expectedErr: false,
}, },
{ {
name: "with-partial-tls-config", name: "custom-read-buffer-size",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "", PrivateKeyFile: "../../testdata/cert.key"}}, cfg: &Config{ReadBufferSize: 65536},
expectedAddress: "0.0.0.0", expectedAddress: "0.0.0.0",
expectedPort: 443, expectedPort: 8080,
expectedErr: true, expectedReadBufferSize: 65536,
expectedErr: false,
},
{
name: "with-good-tls-config",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/cert.pem", PrivateKeyFile: "../../testdata/cert.key"}},
expectedAddress: "0.0.0.0",
expectedPort: 443,
expectedReadBufferSize: 8192,
expectedErr: false,
},
{
name: "with-bad-tls-config",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "../../testdata/badcert.pem", PrivateKeyFile: "../../testdata/cert.key"}},
expectedAddress: "0.0.0.0",
expectedPort: 443,
expectedReadBufferSize: 8192,
expectedErr: true,
},
{
name: "with-partial-tls-config",
cfg: &Config{Port: 443, TLS: &TLSConfig{CertificateFile: "", PrivateKeyFile: "../../testdata/cert.key"}},
expectedAddress: "0.0.0.0",
expectedPort: 443,
expectedReadBufferSize: 8192,
expectedErr: true,
}, },
} }
for _, scenario := range scenarios { for _, scenario := range scenarios {
@ -68,10 +100,13 @@ func TestConfig_ValidateAndSetDefaults(t *testing.T) {
} }
if !scenario.expectedErr { if !scenario.expectedErr {
if scenario.cfg.Port != scenario.expectedPort { if scenario.cfg.Port != scenario.expectedPort {
t.Errorf("expected port to be %d, got %d", scenario.expectedPort, scenario.cfg.Port) t.Errorf("expected Port to be %d, got %d", scenario.expectedPort, scenario.cfg.Port)
}
if scenario.cfg.ReadBufferSize != scenario.expectedReadBufferSize {
t.Errorf("expected ReadBufferSize to be %d, got %d", scenario.expectedReadBufferSize, scenario.cfg.ReadBufferSize)
} }
if scenario.cfg.Address != scenario.expectedAddress { if scenario.cfg.Address != scenario.expectedAddress {
t.Errorf("expected address to be %s, got %s", scenario.expectedAddress, scenario.cfg.Address) t.Errorf("expected Address to be %s, got %s", scenario.expectedAddress, scenario.cfg.Address)
} }
} }
}) })