Merge branch 'master' into 'allow-disabling-redis'

# Conflicts:
#   charts/funkwhale/templates/_helpers.tpl
#   charts/funkwhale/values.yaml

charts/element-web/Chart.yaml

@@ -7,10 +7,10 @@ description: |
   Talk to everyone through the open global Matrix network, protected by proper
   end-to-end encryption.
 icon: https://element.io/images/element-logo.svg
-appVersion: 1.7.22
+appVersion: 1.7.23
 
 type: application
-version: 1.0.17
+version: 1.0.18
 
 maintainers:
   - name: Alexander Olofsson

charts/funkwhale/Chart.yaml

@@ -1,7 +1,7 @@
 ---
 apiVersion: v1
-appVersion: "1.0.1"
+appVersion: "1.1"
 description: A social platform to enjoy and share music
 icon: https://funkwhale.audio/favicon.png
 name: funkwhale
-version: 0.3.6
+version: 0.3.7

charts/funkwhale/templates/_helpers.tpl

@@ -10,6 +10,10 @@ Expand the name of the chart.
 postgres://{{ .Values.database.user }}:{{ .Values.database.password }}@{{ .Values.database.host }}:{{ .Values.database.port }}/{{ .Values.database.database }}
 {{- end -}}
 
+{{- define "funkwhale.redisUrl" -}}
+redis://:{{ .Values.redis.password }}@{{ .Values.redis.host | default (printf "%s-%s" (include "funkwhale.fullname" .) "redis-master") }}:{{ .Values.redis.redisPort }}/0
+{{- end -}}
+
 {{/*
 Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).

charts/matrix-media-repo/Chart.yaml

@@ -2,10 +2,10 @@
 apiVersion: v2
 name: matrix-media-repo
 description: Matrix media repository with multi-domain in mind.
-appVersion: 1.2.4
+appVersion: 1.2.5
 
 type: application
-version: 1.0.2
+version: 1.0.3
 maintainers:
   - name: Alexander Olofsson
     email: ace@haxalot.com

charts/matrix-media-repo/values.yaml

@@ -43,6 +43,9 @@ config:
 # featureSupport:
 #   MSC2448:
 #     enabled: true
+# sentry:
+#   enabled: true
+#   dsn: "https://examplekey@ingest.sentry.io/0"
 
 ## For setting extra parameters on the repo block, separated to avoid breaking
 ## the defaults when merging multiple configurations.

charts/matrix-synapse/Chart.yaml

@@ -6,7 +6,7 @@ icon: https://matrix.org/images/matrix-logo.svg
 appVersion: 1.29.0
 
 type: application
-version: 1.4.1
+version: 2.0.0
 maintainers:
   - name: Alexander Olofsson
     email: ace@haxalot.com

charts/matrix-synapse/scripts/signing-key.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -eu
+
+check_key() {
+  set +e
+
+  echo "Checking for existing signing key..."
+  key="$(kubectl get secret "$SECRET_NAME" -o jsonpath="{.data['signing\.key']}" 2> /dev/null)"
+  [ $? -ne 0 ] && return 1
+  [ -z "$key" ] && return 2
+  return 0
+}
+
+create_key() {
+  echo "Waiting for new signing key to be generated..."
+  begin=$(date +%s)
+  end=$((begin + 300)) # 5 minutes
+  while true; do
+    [ -f /synapse/keys/signing.key ] && return 0
+    [ "$(date +%s)" -gt $end ] && return 1
+    sleep 5
+  done
+}
+
+store_key() {
+  echo "Storing signing key in Kubernetes secret..."
+  kubectl patch secret "$SECRET_NAME" -p "{\"data\":{\"signing.key\":\"$(base64 /synapse/keys/signing.key | tr -d '\n')\"}}"
+}
+
+if check_key; then
+  echo "Key already in place, exiting."
+  exit
+fi
+
+if ! create_key; then
+  echo "Timed out waiting for a signing key to appear."
+  exit 1
+fi
+
+store_key

charts/matrix-synapse/templates/_helpers.tpl

@@ -59,7 +59,7 @@ Create chart name and version as used by the chart label.
 Get the correct image tag name
 */}}
 {{- define "matrix-synapse.imageTag" -}}
-{{- .Values.image.tag | default (printf "%s" .Chart.AppVersion) -}}
+{{- .Values.image.tag | default (printf "v%s" .Chart.AppVersion) -}}
 {{- end -}}
 
 {{/*

charts/matrix-synapse/templates/configuration-scripts.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "matrix-synapse.fullname" . }}-scripts
+  labels:
+    {{- include "matrix-synapse.labels" . | nindent 4 }}
+data:
+{{ (.Files.Glob "scripts/*.sh").AsConfig | indent 2 }}

charts/matrix-synapse/templates/deployment.yaml

@@ -1,4 +1,10 @@
 {{- $needsVolumePermissions := and .Values.volumePermissions.enabled (or .Values.persistence.enabled .Values.persistence.existingClaim) }}
+{{- if (and .Values.postgresql.enabled (not .Values.postgresql.postgresqlPassword)) -}}
+{{-  fail "You must specify a static postgres password if using the included postgres chart" -}}
+{{- end -}}
+{{- if (and .Values.redis.enabled (and .Values.redis.usePassword (not .Values.redis.password))) -}}
+{{-  fail "You must specify a static redis password if using the included redis chart" -}}
+{{- end -}}
 ---
 # Server: {{ required "A valid serverName is required" .Values.serverName }}
 apiVersion: apps/v1
@@ -68,9 +74,9 @@ spec:
               {{- end }}
               {{- end }}
 
-              /matrix-synapse $@
+              exec python -B -m synapse.app.homeserver \
-          args:
+                          -c /synapse/config/homeserver.yaml \
-            - synapse.app.homeserver
+                          -c /synapse/config/conf.d/
           env:
 {{- $postgresPass := include "matrix-synapse.postgresql.password" . }}
           {{- if and .Values.postgresql.enabled (not $postgresPass) }}
@@ -116,6 +122,8 @@ spec:
           volumeMounts:
             - name: config
               mountPath: /synapse/config
+            - name: scripts
+              mountPath: /opt/k8s-synapse
             - name: tmpconf
               mountPath: /synapse/config/conf.d
             - name: secrets
@@ -133,6 +141,10 @@ spec:
         - name: config
           configMap:
             name: {{ include "matrix-synapse.fullname" . }}
+        - name: scripts
+          configMap:
+            name: {{ include "matrix-synapse.fullname" . }}-scripts
+            defaultMode: 0755
         - name: secrets
           secret:
             secretName: {{ include "matrix-synapse.fullname" . }}

charts/matrix-synapse/templates/ingress.yaml

@@ -109,17 +109,21 @@ spec:
         {{- end }}
 
         {{- if has . $wkHosts }}
+        {{-   if $.Values.wellknown.enabled }}
           - path: /.well-known/matrix/client
             backend:
-              serviceName: {{ $fullName }}
+              serviceName: {{ $wkName }}
-              servicePort: 8008
+              servicePort: 80
-
-        {{- if $.Values.wellknown.enabled }}
           - path: /.well-known/matrix/server
             backend:
               serviceName: {{ $wkName }}
               servicePort: 80
-        {{- end }}
+        {{-   else }}
+          - path: /.well-known/matrix/client
+            backend:
+              serviceName: {{ $fullName }}
+              servicePort: 8008
+        {{-   end }}
         {{- end }}
   {{- end }}
 {{- end }}

charts/matrix-synapse/templates/signing-key-job.yaml

@@ -68,18 +68,14 @@ spec:
             - sh
             - -c
             - |
-              echo "Copying key upload script..."
-              cp /key-upload /scripts/
               echo "Generating signing key..."
-              /usr/local/bin/generate_signing_key.py -o /synapse/keys/signing.key
+              generate_signing_key.py -o /synapse/keys/signing.key
-          image: "{{ .Values.image.repository }}:{{ include "matrix-synapse.imageTag" . }}"
+          image: "{{ .Values.signingkey.job.generateImage.repository }}:{{ default .Values.signingkey.job.generateImage.tag "latest" }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          imagePullPolicy: {{ .Values.signingkey.job.generateImage.pullPolicy }}
           name: signing-key-generate
           resources:
             {{- toYaml .Values.signingkey.resources | nindent 12 }}
           volumeMounts:
-            - mountPath: /scripts
-              name: scripts
             - mountPath: /synapse/keys
               name: matrix-synapse-keys
         - command:
@@ -88,13 +84,12 @@ spec:
             - |
               printf "Checking rights to update secret... "
               kubectl auth can-i update secret/${SECRET_NAME}
-              echo "Waiting for key upload script"
+              /scripts/signing-key.sh
-              while ! [ -f /scripts/key-upload ]; do sleep 1; done
-              /scripts/key-upload
           env:
             - name: SECRET_NAME
               value: {{ $secretName }}
-          image: bitnami/kubectl
+          image: "{{ .Values.signingkey.job.publishImage.repository }}:{{ default .Values.signingkey.job.publishImage.tag "latest" }}"
+          imagePullPolicy: {{ .Values.signingkey.job.publishImage.pullPolicy }}
           name: signing-key-upload
           resources:
             {{- toYaml .Values.signingkey.resources | nindent 12 }}
@@ -109,7 +104,12 @@ spec:
       serviceAccount: {{ $name }}
       volumes:
         - name: scripts
-          emptyDir: {}
+          configMap:
+            name: {{ include "matrix-synapse.fullname" . }}-scripts
+            defaultMode: 0755
         - name: matrix-synapse-keys
           emptyDir: {}
+  parallelism: 1
+  completions: 1
+  backoffLimit: 1
 {{- end }}

charts/matrix-synapse/templates/well-known-config.yaml

@@ -23,12 +23,11 @@ data:
     server.groupname     = "lighttpd"
     server.document-root = {{ .Values.wellknown.htdocsPath | quote }}
     server.pid-file      = "/run/lighttpd.pid"
-    server.errorlog      = "/dev/stderr"
     url.rewrite-once     = (
+      "^/\.well-known/matrix/client" => "/client.json",
       "^/\.well-known/matrix/server" => "/server.json"
     )
     status.status-url    = "/server-status"
-    accesslog.filename   = "/dev/stderr"
     extforward.forwarder = ( "all" => "trust")
     setenv.add-response-header = (
       "access-control-allow-headers" => "Origin, X-Requested-With, Content-Type, Accept, Authorization",
@@ -39,5 +38,15 @@ data:
       "content-type" => "application/json"
     )
   server.json: |-
+{{-   if .Values.wellknown.server }}
+{{ toJson .Values.wellknown.server | nindent 4 }}
+{{-   else }}
 {{ dict "m.server" (printf "%s:%d" (.Values.wellknown.host | default (.Values.publicServerName | default .Values.serverName)) (.Values.wellknown.port | default 443)) | toJson | indent 4 }}
-{{- end }}
+{{-   end }}
+  client.json: |-
+{{-   if .Values.wellknown.client }}
+{{ toJson .Values.wellknown.client | nindent 4 }}
+{{-   else }}
+{{ dict "m.homeserver" (dict "base_url" (printf "https://%s/" (.Values.publicServerName | default .Values.serverName))) | toJson | indent 4 }}
+{{-   end }}
+{{- end -}}

charts/matrix-synapse/templates/well-known.yaml

@@ -64,6 +64,9 @@ spec:
             - mountPath: /etc/lighttpd/lighttpd.conf
               name: files
               subPath: lighttpd.conf
+            - mountPath: {{ .Values.wellknown.htdocsPath }}/client.json
+              name: files
+              subPath: client.json
             - mountPath: {{ .Values.wellknown.htdocsPath }}/server.json
               name: files
               subPath: server.json

charts/matrix-synapse/templates/worker-configuration.yaml

@@ -21,7 +21,13 @@ data:
     {{- $name := $worker | replace "_" "-" }}
 
   {{ $name }}.worker: |
-    worker_app: "synapse.app.{{ $worker }}"
+    worker_app: "synapse.app.{{ (not (not $config.generic)) | ternary "generic_worker" $worker }}"
+{{- if $config.name -}}
+{{-   if (ne $config.replicaCount 1) -}}
+{{-     fail "Replica count must be 1 if a worker has a unique name." -}}
+{{-   end }}
+    worker_name: {{ $config.name }}
+{{- end }}
 
     worker_main_http_uri: http://{{ include "matrix-synapse.fullname" $ }}:8008
     worker_replication_host: {{ include "matrix-synapse.replicationname" $ | quote }}
@@ -45,11 +51,17 @@ data:
         x_forwarded: true
 
         resources:
-          - names: {{- toYaml $config.listeners | nindent 14 }}
+          - names:
+              {{- toYaml $config.listeners | nindent 14 }}
             compress: false
     {{- end }}
 
     worker_log_config: /synapse/config/log.yaml
+    {{- if $config.extraConfig }}
+
+    # Extra config
+    {{ toYaml $config.extraConfig | nindent 4 }}
+    {{- end }}
   {{- end }}
 {{- end }}
 {{- end }}

charts/matrix-synapse/templates/worker-deployment.yaml

@@ -20,7 +20,8 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: {{ include (print $.Template.BasePath "/worker-configuration.yaml") $ | sha256sum }}
+        checksum/config: {{ include (print $.Template.BasePath "/configuration.yaml") $ | sha256sum }}
+        checksum/worker-config: {{ include (print $.Template.BasePath "/worker-configuration.yaml") $ | sha256sum }}
         checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") $ | sha256sum }}
       {{- with ($config.annotations | default $default.annotations) }}
         {{ . | toYaml | nindent 8 }}
@@ -62,7 +63,10 @@ spec:
                     -e "s/@@REDIS_PASSWORD@@/${REDIS_PASSWORD:-}/" \
                        > /synapse/config/conf.d/secrets.yaml
 
-              /matrix-synapse synapse.app.{{ $worker }} -c /synapse/config/{{ $name }}.worker
+              exec python -B -m synapse.app.{{ (not (not $config.generic)) | ternary "generic_worker" $worker }} \
+                          -c /synapse/config/homeserver.yaml \
+                          -c /synapse/config/conf.d/ \
+                          -c /synapse/config/{{ $name }}.worker
           env:
           {{- if $.Values.postgresql.enabled }}
             - name: POSTGRES_PASSWORD
@@ -83,7 +87,7 @@ spec:
           {{- end }}
           securityContext:
             {{- $config.securityContext | default $default.securityContext | toYaml | nindent 12 }}
-          image: "{{ $.Values.image.repository }}:{{ $.Chart.AppVersion }}"
+          image: "{{ $.Values.image.repository }}:{{ include "matrix-synapse.imageTag" $ }}"
           imagePullPolicy: {{ $.Values.image.pullPolicy }}
           ports:
             - name: metrics

charts/matrix-synapse/values.yaml

@@ -2,7 +2,7 @@
 ## Docker image configuration, used for Synapse and workers.
 ##
 image:
-  repository: ananace/matrix-synapse
+  repository: matrixdotorg/synapse
   ## Tag to override with, will default to the application version.
   ##
   # tag: ''
@@ -31,6 +31,16 @@ signingkey:
   job:
     enabled: true
 
+    generateImage:
+      repository: matrixdotorg/synapse
+      #tag: latest
+      pullPolicy: IfNotPresent
+
+    publishImage:
+      repository: bitnami/kubectl
+      # tag: latest
+      pullPolicy: IfNotPresent
+
   ## Specify an existing signing key secret, will need to be created in advance.
   ##
   # existingSecret:
@@ -156,7 +166,7 @@ synapse:
 
   ## Additional volumes to mount into Synapse
   ##
-  extraVolumes: {}
+  extraVolumes: []
   #  - name: spamcheck
   #    flexVolume:
   #      driver: ananace/git-live
@@ -164,7 +174,7 @@ synapse:
   #        repo: https://github.com/company/synapse-module
   #        interval: 1d
   #      readOnly: true
-  extraVolumeMounts: {}
+  extraVolumeMounts: []
   #  - name: spamcheck
   #    mountPath: /usr/local/lib/python3.7/site-packages/company
 
@@ -236,6 +246,13 @@ workers:
     ##
     replicaCount: 1
 
+    ## A specific name for this worker, can't be set globally.
+    ## Note that this can only be set when replicaCount is 1
+    #name: 
+    
+    ## Additional configuration to set for the worker, can't be set globally.
+    #extraConfig: {}
+
     ## Annotations to apply to the worker.
     ##
     annotations: {}
@@ -254,8 +271,8 @@ workers:
     ## Additional volumes to add to the worker.
     ## Useful for the media repo, or for adding Python modules.
     ##
-    volumes: {}
+    volumes: []
-    volumeMounts: {}
+    volumeMounts: []
 
     ## Security context information to set to the worker.
     ##
@@ -303,6 +320,7 @@ workers:
   ##
   generic_worker:
     enabled: false
+    generic: true
     listeners: [client, federation]
     csPaths:
     # - "/_matrix/client/(v2_alpha|r0)/sync"
@@ -350,9 +368,34 @@ workers:
       - "/_matrix/federation/v1/event_auth/"
       - "/_matrix/federation/v1/exchange_third_party_invite/"
       - "/_matrix/federation/v1/user/devices/"
+      - "/_matrix/federation/v1/send/"
       - "/_matrix/federation/v1/get_groups_publicised"
       - "/_matrix/key/v2/query"
-      - "/_matrix/federation/v1/send/"
+
+  ## To separate the generic worker into specific concerns - for example federation transaction receiving;
+  ## NB; This worker should have incoming traffic routed based on source IP, which is
+  ##     left as an exercise to the reader.
+  ## https://github.com/matrix-org/synapse/blob/develop/docs/workers.md#load-balancing
+  #federation_reader:
+  #  enabled: true
+  #  generic: true
+  #  listeners: [federation]
+  #  paths:
+  #     - "/_matrix/federation/v1/send/"
+
+  ## Or /sync handling.
+  ## NB; Care should be taken to route users to the same instance when scaling this worker,
+  ##     this is left as an exercise to the reader.
+  ## https://github.com/matrix-org/synapse/blob/develop/docs/workers.md#load-balancing
+  #synchrotron:
+  #  enabled: true
+  #  generic: true
+  #  listeners: [client]
+  #  csPaths:
+  #    - "/_matrix/client/(v2_alpha|r0)/sync"
+  #    - "/_matrix/client/(api/v1|v2_alpha|r0)/events"
+  #    - "/_matrix/client/(api/v1|r0)/initialSync"
+  #    - "/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync"
 
   ## This worker deals with pushing notifications.
   ## NB; Only one instance of this worker can be run at a time, refer to the
@@ -418,8 +461,14 @@ wellknown:
 
   ## The host and port combo to serve on .well-known/matrix/server.
   ##
-  # host: matrix.example.com
+  server: {}
-  # port: 443
+  #  m.server: matrix.example.com:443
+
+  ## Data to serve on .well-known/matrix/client.
+  ##
+  client: {}
+  #  m.homeserver:
+  #    base_url: https://matrix.example.com
 
   ## A custom htdocs path, useful when running another image.
   ##
@@ -485,13 +534,17 @@ wellknown:
 postgresql:
   enabled: true
 
+  # XXX Change me!
+  postgresqlPassword: synapse
+
   postgresqlUsername: synapse
   postgresqlDatabase: synapse
 
   postgresqlInitdbArgs: "--lc-collate=C --lc-ctype=C"
 
   persistence:
-    size: 16G
+    # storageClass: "-"
+    size: 16Gi
 
 ## An externally configured Postgres server to use for Synapse's database, note
 ## that the database needs to have both COLLATE and CTYPE set to "C".
@@ -511,8 +564,10 @@ externalPostgresql:
 redis:
   enabled: true
 
-  # usePassword: false
+  # XXX Change me!
-  # password: synapse
+  usePassword: true
+  password: synapse
+
   cluster:
     enabled: false
   master:

charts/netbox/Chart.yaml

@@ -1,6 +1,6 @@
 ---
 apiVersion: v1
-appVersion: 2.10.5
+appVersion: 2.10.6
 description: |
   An IP address management (IPAM) and data center infrastructure management (DCIM) tool.
 
@@ -8,4 +8,4 @@ description: |
   https://github.com/netbox-community/netbox-docker/releases/tag/1.0.0 for more info.
 icon: https://raw.githubusercontent.com/digitalocean/netbox/develop/netbox/project-static/img/netbox.ico
 name: netbox
-version: 3.0.2
+version: 3.0.3