diff --git a/charts/turbinia/.helmignore b/charts/turbinia/.helmignore new file mode 100644 index 0000000..e746a6c --- /dev/null +++ b/charts/turbinia/.helmignore @@ -0,0 +1 @@ +*/configs/* \ No newline at end of file diff --git a/charts/turbinia/Chart.yaml b/charts/turbinia/Chart.yaml new file mode 100644 index 0000000..1ba1530 --- /dev/null +++ b/charts/turbinia/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: turbinia +description: | + Automation and Scaling of Digital Forensics Tools +appVersion: "20240510" +type: application +version: 0.1.1 +maintainers: +- name: Tommy Skaug + email: tommy@skaug.me +keywords: +- forensics +- google +- scale +- automation +sources: +- https://github.com/google/turbinia +- https://github.com/google/osdfir-infrastructure diff --git a/charts/turbinia/README.md b/charts/turbinia/README.md new file mode 100644 index 0000000..3bc4ea6 --- /dev/null +++ b/charts/turbinia/README.md @@ -0,0 +1,8 @@ + + +## Debugging + +```sh +task flux:sync +kubectl annotate es turbina-conf force-sync=$(date +%s) --overwrite -n sec-forensics +``` diff --git a/charts/turbinia/templates/_helpers.tpl b/charts/turbinia/templates/_helpers.tpl new file mode 100644 index 0000000..f1b5a4b --- /dev/null +++ b/charts/turbinia/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "turbinia.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "turbinia.fullname" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "turbinia.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "turbinia.labels" -}} +helm.sh/chart: {{ include "turbinia.chart" . }} +{{ include "turbinia.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +date: "{{ now | htmlDate }}" +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "turbinia.selectorLabels" -}} +app.kubernetes.io/name: {{ include "turbinia.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the upload path. +*/}} +{{- define "turbinia.uploadPath" -}} +{{- printf "/data/upload" }} +{{- end }} \ No newline at end of file diff --git a/charts/turbinia/templates/deployment-api.yaml b/charts/turbinia/templates/deployment-api.yaml new file mode 100644 index 0000000..caeefb1 --- /dev/null +++ b/charts/turbinia/templates/deployment-api.yaml @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "turbinia.fullname" . }}-api + namespace: {{ .Release.Namespace | quote }} + labels: + app.kubernetes.io/component: api + {{- include "turbinia.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: api + {{- include "turbinia.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + # Have Deployment restart after each upgrade + roll: {{ randAlphaNum 5 | quote }} + prometheus.io/port: {{ .Values.metrics.port | quote }} + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/component: api + {{- include "turbinia.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "turbinia.fullname" . }} + securityContext: + {{- toYaml .Values.api.podSecurityContext | nindent 8 }} + containers: + - name: api + securityContext: + {{- toYaml .Values.api.securityContext | nindent 12 }} + image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.api.image.pullPolicy }} + livenessProbe: + exec: + command: ['sh', '-c', "ps aux | grep api_server | grep -v grep"] + initialDelaySeconds: 5 + periodSeconds: 5 + env: + - name: TURBINIA_EXTRA_ARGS + value: "-d" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /mnt/evidence + name: evidence + - mountPath: /mnt/turbiniavolume + name: turbinia + - mountPath: /etc/turbinia + name: turbinia-conf + ports: + {{- if .Values.metrics.enabled }} + - containerPort: {{ .Values.metrics.port }} + {{- end }} + - containerPort: 8000 + resources: + {{- toYaml .Values.api.resources | nindent 12 }} + volumes: + - name: turbinia + persistentVolumeClaim: + claimName: {{ include "turbinia.fullname" . }} + readOnly: false + - name: evidence + persistentVolumeClaim: + claimName: {{ include "turbinia.fullname" . }}-evidence + readOnly: false + - name: turbinia-conf + secret: + secretName: {{ include "turbinia.fullname" . }}-conf + defaultMode: 0744 + {{- with .Values.api.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.api.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.api.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/charts/turbinia/templates/deployment-controller.yaml b/charts/turbinia/templates/deployment-controller.yaml new file mode 100644 index 0000000..e946175 --- /dev/null +++ b/charts/turbinia/templates/deployment-controller.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "turbinia.fullname" . }}-controller + namespace: {{ .Release.Namespace | quote }} + labels: + app.kubernetes.io/component: controller + {{- include "turbinia.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: controller + {{- include "turbinia.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + prometheus.io/port: {{ .Values.metrics.port | quote }} + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/component: controller + {{- include "turbinia.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "turbinia.fullname" . }} + securityContext: + {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + containers: + - name: controller + securityContext: + {{- toYaml .Values.controller.securityContext | nindent 12 }} + image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + lifecycle: + preStop: + exec: + command: + - "/bin/sh" + - "-c" + - "touch /tmp/turbinia-to-scaledown.lock && sleep 5 && /usr/bin/python3 /home/turbinia/check-lockfile.py" + env: + - name: TURBINIA_EXTRA_ARGS + value: "-d" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /mnt/evidence + name: evidence + - mountPath: /mnt/turbiniavolume + name: turbinia + - mountPath: /etc/turbinia + name: turbinia-conf + ports: + - containerPort: {{ .Values.metrics.port }} + resources: + {{- toYaml .Values.controller.resources | nindent 12 }} + volumes: + - name: turbinia + persistentVolumeClaim: + claimName: {{ include "turbinia.fullname" . }} + readOnly: false + - name: evidence + persistentVolumeClaim: + claimName: {{ include "turbinia.fullname" . }}-evidence + readOnly: false + - name: turbinia-conf + secret: + secretName: {{ include "turbinia.fullname" . }}-conf + defaultMode: 0744 + {{- with .Values.controller.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/charts/turbinia/templates/deployment-server.yaml b/charts/turbinia/templates/deployment-server.yaml new file mode 100644 index 0000000..c43b75d --- /dev/null +++ b/charts/turbinia/templates/deployment-server.yaml @@ -0,0 +1,86 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "turbinia.fullname" . }}-server + namespace: {{ .Release.Namespace | quote }} + labels: + app.kubernetes.io/component: server + {{- include "turbinia.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: server + {{- include "turbinia.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + # Have Deployment restart after each upgrade + roll: {{ randAlphaNum 5 | quote }} + prometheus.io/port: {{ .Values.metrics.port | quote }} + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/component: server + {{- include "turbinia.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "turbinia.fullname" . }} + securityContext: + {{- toYaml .Values.server.podSecurityContext | nindent 8 }} + initContainers: + - name: init-permissions + image: busybox + command: ['sh', '-c', 'chmod go+w /mnt/evidence && chmod go+w /mnt/turbiniavolume && mkdir -p /etc/turbinia'] + volumeMounts: + - name: turbinia + mountPath: /mnt/turbiniavolume + - name: evidence + mountPath: /mnt/evidence + containers: + - name: server + securityContext: + {{- toYaml .Values.server.securityContext | nindent 12 }} + image: "{{ .Values.server.image.repository }}:{{ .Values.server.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.server.image.pullPolicy }} + env: + - name: TURBINIA_EXTRA_ARGS + value: "-d" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /mnt/evidence + name: evidence + - mountPath: /mnt/turbiniavolume + name: turbinia + - mountPath: /etc/turbinia + name: turbinia-conf + ports: + - containerPort: {{ .Values.metrics.port }} + resources: + {{- toYaml .Values.server.resources | nindent 12 }} + volumes: + - name: turbinia + persistentVolumeClaim: + claimName: {{ include "turbinia.fullname" . }} + readOnly: false + - name: evidence + persistentVolumeClaim: + claimName: {{ include "turbinia.fullname" . }}-evidence + readOnly: false + - name: turbinia-conf + secret: + secretName: {{ include "turbinia.fullname" . }}-conf + defaultMode: 0744 + {{- with .Values.server.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/charts/turbinia/templates/deployment-worker.yaml b/charts/turbinia/templates/deployment-worker.yaml new file mode 100644 index 0000000..4fa500f --- /dev/null +++ b/charts/turbinia/templates/deployment-worker.yaml @@ -0,0 +1,95 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "turbinia.fullname" . }}-worker + namespace: {{ .Release.Namespace | quote }} + labels: + app.kubernetes.io/component: worker + {{- include "turbinia.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/component: worker + {{- include "turbinia.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + # Have Deployment restart after each upgrade + roll: {{ randAlphaNum 5 | quote }} + prometheus.io/port: {{ .Values.metrics.port | quote }} + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/component: worker + {{- include "turbinia.selectorLabels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "turbinia.fullname" . }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + # The grace period needs to be set to the largest task timeout as + # set in the turbinia configuration file. + terminationGracePeriodSeconds: 86400 + initContainers: + - name: init-permissions + image: busybox + command: ['sh', '-c', 'chmod go+w /mnt/evidence && chmod go+w /mnt/turbiniavolume && mkdir -p /etc/turbinia'] + volumeMounts: + - name: turbinia + mountPath: /mnt/turbiniavolume + - name: evidence + mountPath: /mnt/evidence + containers: + - name: worker + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + lifecycle: + preStop: + exec: + command: + - "/bin/sh" + - "-c" + - "touch /tmp/turbinia-to-scaledown.lock && sleep 5 && /usr/bin/python3 /home/turbinia/check-lockfile.py" + env: + - name: TURBINIA_EXTRA_ARGS + value: "-d" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /mnt/evidence + name: evidence + - mountPath: /mnt/turbiniavolume + name: turbinia + - mountPath: /etc/turbinia + name: turbinia-conf + ports: + - containerPort: {{ .Values.metrics.port }} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + volumes: + - name: turbinia + persistentVolumeClaim: + claimName: {{ include "turbinia.fullname" . }} + readOnly: false + - name: evidence + persistentVolumeClaim: + claimName: {{ include "turbinia.fullname" . }}-evidence + readOnly: false + - name: turbinia-conf + secret: + secretName: {{ include "turbinia.fullname" . }}-conf + defaultMode: 0744 + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/turbinia/templates/hpa.yaml b/charts/turbinia/templates/hpa.yaml new file mode 100644 index 0000000..2ddb395 --- /dev/null +++ b/charts/turbinia/templates/hpa.yaml @@ -0,0 +1,14 @@ +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "turbinia.fullname" . }} + labels: + {{- include "turbinia.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "turbinia.fullname" . }}-worker + minReplicas: {{ .Values.worker.autoscaling.minReplicas }} + maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }} + targetCPUUtilizationPercentage: {{ .Values.worker.autoscaling.targetCPUUtilizationPercentage }} \ No newline at end of file diff --git a/charts/turbinia/templates/pvc.yaml b/charts/turbinia/templates/pvc.yaml new file mode 100644 index 0000000..ada99d4 --- /dev/null +++ b/charts/turbinia/templates/pvc.yaml @@ -0,0 +1,41 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ include "turbinia.fullname" . }} + labels: + {{- include "turbinia.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.general.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.general.size | quote }} +{{- if .Values.persistence.general.storageClass }} +{{- if (eq "-" .Values.persistence.general.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.persistence.general.storageClass }}" +{{- end }} +{{- end }} +--- +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ include "turbinia.fullname" . }}-evidence + labels: + {{- include "turbinia.labels" . | nindent 4 }} +spec: + accessModes: + - {{ .Values.persistence.evidence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.evidence.size | quote }} +{{- if .Values.persistence.evidence.storageClass }} +{{- if (eq "-" .Values.persistence.evidence.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.persistence.evidence.storageClass }}" +{{- end }} +{{- end }} diff --git a/charts/turbinia/templates/service.yaml b/charts/turbinia/templates/service.yaml new file mode 100644 index 0000000..bd55595 --- /dev/null +++ b/charts/turbinia/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "turbinia.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "turbinia.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 8000 + protocol: TCP + targetPort: 8000 + selector: + app.kubernetes.io/component: server + {{- include "turbinia.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/charts/turbinia/templates/serviceaccount.yaml b/charts/turbinia/templates/serviceaccount.yaml new file mode 100644 index 0000000..f91de5b --- /dev/null +++ b/charts/turbinia/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "turbinia.fullname" . }} + labels: + {{- include "turbinia.labels" . | nindent 4 }} \ No newline at end of file diff --git a/charts/turbinia/values.yaml b/charts/turbinia/values.yaml new file mode 100644 index 0000000..17406b0 --- /dev/null +++ b/charts/turbinia/values.yaml @@ -0,0 +1,93 @@ +config: + externalUrl: https://turbinia.example.com/ + +persistence: + general: + storageClass: ceph-filesystem + accessMode: ReadWriteMany + size: 2Gi + evidence: + storageClass: ceph-filesystem + accessMode: ReadWriteMany + size: 2Gi + +server: + image: + repository: us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-server + pullPolicy: IfNotPresent + tag: "20240510" + podSecurityContext: {} + securityContext: {} + nodeSelector: {} + tolerations: [] + affinity: {} + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 50m + memory: 128Mi + +worker: + image: + repository: us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker + pullPolicy: IfNotPresent + tag: "20240510" + podSecurityContext: {} + securityContext: + # Due to Turbinia attaching and detaching disks, a privileged container is required for the worker container. + privileged: true + nodeSelector: {} + tolerations: [] + affinity: {} + autoscaling: + minReplicas: 1 + maxReplicas: 2 + targetCPUUtilizationPercentage: 50 + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 50m + memory: 128Mi + +api: + image: + repository: us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-api-server + pullPolicy: IfNotPresent + tag: "20240510" + podSecurityContext: {} + securityContext: {} + nodeSelector: {} + tolerations: [] + affinity: {} + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 50m + memory: 128Mi + +controller: + image: + repository: us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-controller + pullPolicy: IfNotPresent + tag: "20240510" + podSecurityContext: {} + securityContext: {} + nodeSelector: {} + tolerations: [] + affinity: {} + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 50m + memory: 128Mi + +metrics: + port: 9001