feat: add turbinia

charts/turbinia/.helmignore

@@ -0,0 +1 @@
+*/configs/*

charts/turbinia/Chart.yaml

@@ -0,0 +1,18 @@
+apiVersion: v2
+name: turbinia
+description: |
+  Automation and Scaling of Digital Forensics Tools
+appVersion: "20240510"
+type: application
+version: 0.1.1
+maintainers:
+- name: Tommy Skaug
+  email: tommy@skaug.me
+keywords:
+- forensics
+- google
+- scale
+- automation
+sources:
+- https://github.com/google/turbinia
+- https://github.com/google/osdfir-infrastructure

charts/turbinia/README.md

@@ -0,0 +1,8 @@
+
+
+## Debugging
+
+```sh
+task flux:sync
+kubectl annotate es turbina-conf force-sync=$(date +%s) --overwrite -n sec-forensics
+```

charts/turbinia/templates/_helpers.tpl

@@ -0,0 +1,45 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "turbinia.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "turbinia.fullname" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "turbinia.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "turbinia.labels" -}}
+helm.sh/chart: {{ include "turbinia.chart" . }}
+{{ include "turbinia.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+date: "{{ now | htmlDate }}"
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "turbinia.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "turbinia.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the upload path.
+*/}}
+{{- define "turbinia.uploadPath" -}}
+{{- printf "/data/upload" }}
+{{- end }}

charts/turbinia/templates/deployment-api.yaml

@@ -0,0 +1,85 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name:  {{ include "turbinia.fullname" . }}-api
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    app.kubernetes.io/component: api
+    {{- include "turbinia.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: api
+      {{- include "turbinia.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        # Have Deployment restart after each upgrade
+        roll: {{ randAlphaNum 5 | quote }}
+        prometheus.io/port: {{ .Values.metrics.port | quote }}
+        prometheus.io/scrape: "true"
+      labels:
+        app.kubernetes.io/component: api
+        {{- include "turbinia.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "turbinia.fullname" . }}
+      securityContext:
+        {{- toYaml .Values.api.podSecurityContext | nindent 8 }}
+      containers:
+        - name: api
+          securityContext:
+              {{- toYaml .Values.api.securityContext | nindent 12 }}
+          image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.api.image.pullPolicy }}
+          livenessProbe:
+            exec:
+              command: ['sh', '-c', "ps aux | grep api_server | grep -v grep"]
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          env:
+            - name: TURBINIA_EXTRA_ARGS
+              value: "-d"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /mnt/evidence
+              name: evidence
+            - mountPath: /mnt/turbiniavolume
+              name: turbinia
+            - mountPath: /etc/turbinia
+              name: turbinia-conf
+          ports:
+            {{- if .Values.metrics.enabled }}
+            - containerPort: {{ .Values.metrics.port }}
+            {{- end }}
+            - containerPort: 8000
+          resources:
+            {{- toYaml .Values.api.resources | nindent 12 }}
+      volumes:
+        - name: turbinia
+          persistentVolumeClaim:
+            claimName: {{ include "turbinia.fullname" . }}
+            readOnly: false
+        - name: evidence
+          persistentVolumeClaim:
+            claimName: {{ include "turbinia.fullname" . }}-evidence
+            readOnly: false
+        - name: turbinia-conf
+          secret:
+            secretName: {{ include "turbinia.fullname" . }}-conf
+            defaultMode: 0744
+      {{- with .Values.api.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.api.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.api.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/turbinia/templates/deployment-controller.yaml

@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name:  {{ include "turbinia.fullname" . }}-controller
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    app.kubernetes.io/component: controller
+    {{- include "turbinia.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: controller
+      {{- include "turbinia.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: {{ .Values.metrics.port | quote }}
+        prometheus.io/scrape: "true"
+      labels:
+        app.kubernetes.io/component: controller
+        {{- include "turbinia.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "turbinia.fullname" . }}
+      securityContext:
+        {{- toYaml .Values.controller.podSecurityContext | nindent 8 }}
+      containers:
+        - name: controller
+          securityContext:
+              {{- toYaml .Values.controller.securityContext | nindent 12 }}
+          image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.controller.image.pullPolicy }}
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - "/bin/sh"
+                  - "-c"
+                  - "touch /tmp/turbinia-to-scaledown.lock && sleep 5 && /usr/bin/python3 /home/turbinia/check-lockfile.py"
+          env:
+            - name: TURBINIA_EXTRA_ARGS
+              value: "-d"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /mnt/evidence
+              name: evidence
+            - mountPath: /mnt/turbiniavolume
+              name: turbinia
+            - mountPath: /etc/turbinia
+              name: turbinia-conf
+          ports:
+            - containerPort: {{ .Values.metrics.port }}
+          resources:
+            {{- toYaml .Values.controller.resources | nindent 12 }}
+      volumes:
+        - name: turbinia
+          persistentVolumeClaim:
+            claimName: {{ include "turbinia.fullname" . }}
+            readOnly: false
+        - name: evidence
+          persistentVolumeClaim:
+            claimName: {{ include "turbinia.fullname" . }}-evidence
+            readOnly: false
+        - name: turbinia-conf
+          secret:
+            secretName: {{ include "turbinia.fullname" . }}-conf
+            defaultMode: 0744
+      {{- with .Values.controller.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controller.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controller.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/turbinia/templates/deployment-server.yaml

@@ -0,0 +1,86 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name:  {{ include "turbinia.fullname" . }}-server
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    app.kubernetes.io/component: server
+    {{- include "turbinia.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: server
+      {{- include "turbinia.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        # Have Deployment restart after each upgrade
+        roll: {{ randAlphaNum 5 | quote }}
+        prometheus.io/port: {{ .Values.metrics.port | quote }}
+        prometheus.io/scrape: "true"
+      labels:
+        app.kubernetes.io/component: server
+        {{- include "turbinia.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "turbinia.fullname" . }}
+      securityContext:
+        {{- toYaml .Values.server.podSecurityContext | nindent 8 }}
+      initContainers:
+        - name: init-permissions
+          image: busybox
+          command: ['sh', '-c', 'chmod go+w /mnt/evidence && chmod go+w /mnt/turbiniavolume && mkdir -p /etc/turbinia']
+          volumeMounts:
+            - name: turbinia
+              mountPath: /mnt/turbiniavolume
+            - name: evidence
+              mountPath: /mnt/evidence
+      containers:
+        - name: server
+          securityContext:
+            {{- toYaml .Values.server.securityContext | nindent 12 }}
+          image: "{{ .Values.server.image.repository }}:{{ .Values.server.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.server.image.pullPolicy }}
+          env:
+            - name: TURBINIA_EXTRA_ARGS
+              value: "-d"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /mnt/evidence
+              name: evidence
+            - mountPath: /mnt/turbiniavolume
+              name: turbinia
+            - mountPath: /etc/turbinia
+              name: turbinia-conf
+          ports:
+            - containerPort: {{ .Values.metrics.port }}
+          resources:
+            {{- toYaml .Values.server.resources | nindent 12 }}
+      volumes:
+        - name: turbinia
+          persistentVolumeClaim:
+            claimName: {{ include "turbinia.fullname" . }}
+            readOnly: false
+        - name: evidence
+          persistentVolumeClaim:
+            claimName: {{ include "turbinia.fullname" . }}-evidence
+            readOnly: false
+        - name: turbinia-conf
+          secret:
+            secretName: {{ include "turbinia.fullname" . }}-conf
+            defaultMode: 0744
+      {{- with .Values.server.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.server.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.server.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/turbinia/templates/deployment-worker.yaml

@@ -0,0 +1,95 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name:  {{ include "turbinia.fullname" . }}-worker
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    app.kubernetes.io/component: worker
+    {{- include "turbinia.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: worker
+      {{- include "turbinia.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        # Have Deployment restart after each upgrade
+        roll: {{ randAlphaNum 5 | quote }}
+        prometheus.io/port: {{ .Values.metrics.port | quote }}
+        prometheus.io/scrape: "true"
+      labels:
+        app.kubernetes.io/component: worker
+        {{- include "turbinia.selectorLabels" . | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "turbinia.fullname" . }}
+      securityContext:
+        {{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
+      # The grace period needs to be set to the largest task timeout as
+      # set in the turbinia configuration file.
+      terminationGracePeriodSeconds: 86400
+      initContainers:
+        - name: init-permissions
+          image: busybox
+          command: ['sh', '-c', 'chmod go+w /mnt/evidence && chmod go+w /mnt/turbiniavolume && mkdir -p /etc/turbinia']
+          volumeMounts:
+            - name: turbinia
+              mountPath: /mnt/turbiniavolume
+            - name: evidence
+              mountPath: /mnt/evidence
+      containers:
+        - name: worker
+          securityContext:
+              {{- toYaml .Values.worker.securityContext | nindent 12 }}
+          image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - "/bin/sh"
+                  - "-c"
+                  - "touch /tmp/turbinia-to-scaledown.lock && sleep 5 && /usr/bin/python3 /home/turbinia/check-lockfile.py"
+          env:
+            - name: TURBINIA_EXTRA_ARGS
+              value: "-d"
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /mnt/evidence
+              name: evidence
+            - mountPath: /mnt/turbiniavolume
+              name: turbinia
+            - mountPath: /etc/turbinia
+              name: turbinia-conf
+          ports:
+            - containerPort: {{ .Values.metrics.port }}
+          resources:
+            {{- toYaml .Values.worker.resources | nindent 12 }}
+      volumes:
+        - name: turbinia
+          persistentVolumeClaim:
+            claimName: {{ include "turbinia.fullname" . }}
+            readOnly: false
+        - name: evidence
+          persistentVolumeClaim:
+            claimName: {{ include "turbinia.fullname" . }}-evidence
+            readOnly: false
+        - name: turbinia-conf
+          secret:
+            secretName: {{ include "turbinia.fullname" . }}-conf
+            defaultMode: 0744
+      {{- with .Values.worker.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.worker.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.worker.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

charts/turbinia/templates/hpa.yaml

@@ -0,0 +1,14 @@
+apiVersion: autoscaling/v1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "turbinia.fullname" . }}
+  labels:
+    {{- include "turbinia.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "turbinia.fullname" . }}-worker
+  minReplicas: {{ .Values.worker.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }}
+  targetCPUUtilizationPercentage: {{ .Values.worker.autoscaling.targetCPUUtilizationPercentage }}

charts/turbinia/templates/pvc.yaml

@@ -0,0 +1,41 @@
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ include "turbinia.fullname" . }}
+  labels:
+    {{- include "turbinia.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.general.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.general.size | quote }}
+{{- if .Values.persistence.general.storageClass }}
+{{- if (eq "-" .Values.persistence.general.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .Values.persistence.general.storageClass }}"
+{{- end }}
+{{- end }}
+---
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ include "turbinia.fullname" . }}-evidence
+  labels:
+    {{- include "turbinia.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.evidence.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.evidence.size | quote }}
+{{- if .Values.persistence.evidence.storageClass }}
+{{- if (eq "-" .Values.persistence.evidence.storageClass) }}
+  storageClassName: ""
+{{- else }}
+  storageClassName: "{{ .Values.persistence.evidence.storageClass }}"
+{{- end }}
+{{- end }}

charts/turbinia/templates/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "turbinia.fullname" . }}
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    {{- include "turbinia.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8000
+      protocol: TCP
+      targetPort: 8000
+  selector:
+    app.kubernetes.io/component: server
+    {{- include "turbinia.selectorLabels" . | nindent 4 }}

charts/turbinia/templates/serviceaccount.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "turbinia.fullname" . }}
+  labels:
+    {{- include "turbinia.labels" . | nindent 4 }}

charts/turbinia/values.yaml

@@ -0,0 +1,93 @@
+config:
+  externalUrl: https://turbinia.example.com/
+
+persistence:
+  general:
+    storageClass: ceph-filesystem
+    accessMode: ReadWriteMany
+    size: 2Gi
+  evidence:
+    storageClass: ceph-filesystem
+    accessMode: ReadWriteMany
+    size: 2Gi
+
+server:
+  image:
+    repository: us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-server
+    pullPolicy: IfNotPresent
+    tag: "20240510"
+  podSecurityContext: {}
+  securityContext: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  resources:
+    limits:
+      cpu: 500m
+      memory: 1Gi
+    requests:
+      cpu: 50m
+      memory: 128Mi
+
+worker:
+  image:
+    repository: us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker
+    pullPolicy: IfNotPresent
+    tag: "20240510"
+  podSecurityContext: {}
+  securityContext:
+    # Due to Turbinia attaching and detaching disks, a privileged container is required for the worker container.
+    privileged: true
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  autoscaling:
+    minReplicas: 1
+    maxReplicas: 2
+    targetCPUUtilizationPercentage: 50
+  resources:
+    limits:
+      cpu: 500m
+      memory: 1Gi
+    requests:
+      cpu: 50m
+      memory: 128Mi
+
+api:
+  image:
+    repository: us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-api-server
+    pullPolicy: IfNotPresent
+    tag: "20240510"
+  podSecurityContext: {}
+  securityContext: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  resources:
+    limits:
+      cpu: 500m
+      memory: 1Gi
+    requests:
+      cpu: 50m
+      memory: 128Mi
+
+controller:
+  image:
+    repository: us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-controller
+    pullPolicy: IfNotPresent
+    tag: "20240510"
+  podSecurityContext: {}
+  securityContext: {}
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}
+  resources:
+    limits:
+      cpu: 500m
+      memory: 1Gi
+    requests:
+      cpu: 50m
+      memory: 128Mi
+
+metrics:
+  port: 9001