docs(longhorn): document 2026-04-13 power-cut recovery + add data-recovery tooling
Captures the post-mortem of the April 13 power-cut: incident timeline, retrospective, and architecture/role diagrams. Adds an ADR explaining why Longhorn cannot re-associate orphaned replica directories after a nuclear reinstall (engine-id naming), plus block-device recovery runbooks and the `playbooks/recover/longhorn_data.yml` automation that wires `merge-longhorn-layers.py` to rebuild PVCs from raw `volume-head-*.img` chains. Also extends the k3s_pvc backup to capture Longhorn `volumes`/`settings` CRDs (needed for the fast-path restore) and rewrites the restore script with a fallback dir + English messages. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
Claude Opus 4.7
@@ -24,12 +24,15 @@
|
||||
|
||||
- name: define backup command
|
||||
set_fact:
|
||||
backup_cmd: |-
|
||||
echo "
|
||||
$(kubectl get -A pv -o yaml)
|
||||
---
|
||||
$(kubectl get -A pvc -o yaml)
|
||||
"
|
||||
# PVs + PVCs + Longhorn Volume CRDs (critical for fast recovery — without Volume CRDs,
|
||||
# Longhorn cannot re-associate orphaned replica dirs after a reinstall and forces
|
||||
# full block-device injection recovery. See docs/adr/20260414-longhorn-pvc-recovery.md)
|
||||
backup_cmd: >-
|
||||
kubectl get -A pv,pvc -o yaml
|
||||
&& echo '---'
|
||||
&& kubectl get -A volumes.longhorn.io -o yaml
|
||||
&& echo '---'
|
||||
&& kubectl get -A settings.longhorn.io -o yaml
|
||||
|
||||
- name: test backup_cmd
|
||||
ansible.builtin.shell: |
|
||||
@@ -65,19 +68,34 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
BACKUP_DIR="{{ backup_dir }}"
|
||||
PRIMARY_BACKUP_DIR="{{ backup_dir }}"
|
||||
FALLBACK_BACKUP_DIR="/home/pi/arcodange/backups/k3s_pvc"
|
||||
|
||||
# Check if fallback directory exists and has backups
|
||||
if [ -d "$FALLBACK_BACKUP_DIR" ] && ls "$FALLBACK_BACKUP_DIR"/*.volumes 1>/dev/null 2>&1; then
|
||||
BACKUP_DIR="$FALLBACK_BACKUP_DIR"
|
||||
echo "Using fallback backup directory: $BACKUP_DIR"
|
||||
elif [ -d "$PRIMARY_BACKUP_DIR" ] && ls "$PRIMARY_BACKUP_DIR"/*.volumes 1>/dev/null 2>&1; then
|
||||
BACKUP_DIR="$PRIMARY_BACKUP_DIR"
|
||||
else
|
||||
echo "No backup directory found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
FILE=$(ls -1t "$BACKUP_DIR"/backup_*.volumes | head -n 1)
|
||||
echo "Aucune date fournie, restauration du dernier dump : $FILE"
|
||||
echo "No date provided, restoring latest dump: $FILE"
|
||||
else
|
||||
FILE="$BACKUP_DIR/backup_$1.volumes"
|
||||
if [ ! -f "$FILE" ]; then
|
||||
echo "Fichier $FILE introuvable"
|
||||
echo "File $FILE not found"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
kubectl apply -f "$FILE"
|
||||
|
||||
echo "Restauration des volumes k3s terminée."
|
||||
echo "K3S volumes restoration complete."
|
||||
echo "NOTE: file includes PVs, PVCs, and Longhorn Volume CRDs."
|
||||
echo "If Longhorn replica dirs are still orphaned after this restore,"
|
||||
echo "fall back to: ansible-playbook playbooks/recover/longhorn_data.yml"
|
||||
|
||||
@@ -0,0 +1,536 @@
|
||||
---
|
||||
- name: Recover Longhorn from Power Cut - CSI Driver Registration Loss
|
||||
hosts: raspberries:&local
|
||||
gather_facts: yes
|
||||
become: yes
|
||||
|
||||
vars:
|
||||
# Backup locations
|
||||
primary_backup_dir: "/mnt/backups/k3s_pvc"
|
||||
fallback_backup_dir: "/home/pi/arcodange/backups/k3s_pvc"
|
||||
scripts_dir: "/opt/k3s_volumes"
|
||||
|
||||
# Longhorn configuration
|
||||
longhorn_manifest_path: "/var/lib/rancher/k3s/server/manifests/longhorn-install.yaml"
|
||||
longhorn_namespace: "longhorn-system"
|
||||
longhorn_chart_name: "longhorn-install"
|
||||
longhorn_chart_namespace: "kube-system"
|
||||
|
||||
# Data paths (DO NOT MODIFY - points to actual volume data)
|
||||
longhorn_data_path: "/mnt/arcodange/longhorn"
|
||||
|
||||
tasks:
|
||||
# ========================================================================
|
||||
# PHASE 0: Pre-flight Checks
|
||||
# ========================================================================
|
||||
|
||||
- name: Verify data directory exists on control plane
|
||||
ansible.builtin.stat:
|
||||
path: "{{ longhorn_data_path }}"
|
||||
register: data_dir
|
||||
when: inventory_hostname == 'pi1'
|
||||
run_once: true
|
||||
|
||||
- name: FAIL if data directory missing
|
||||
ansible.builtin.fail:
|
||||
msg: "CRITICAL: Longhorn data directory {{ longhorn_data_path }} does not exist. Aborting recovery."
|
||||
when: inventory_hostname == 'pi1' and not data_dir.stat.exists
|
||||
run_once: true
|
||||
|
||||
- name: Check for fallback backups on pi1
|
||||
ansible.builtin.shell: ls {{ fallback_backup_dir }}/backup_*.volumes 2>/dev/null
|
||||
register: fallback_backup_check
|
||||
changed_when: false
|
||||
when: inventory_hostname == 'pi1'
|
||||
run_once: true
|
||||
ignore_errors: yes
|
||||
|
||||
- name: Check for primary backups on pi1
|
||||
ansible.builtin.shell: ls {{ primary_backup_dir }}/backup_*.volumes 2>/dev/null
|
||||
register: primary_backup_check
|
||||
changed_when: false
|
||||
when: inventory_hostname == 'pi1'
|
||||
run_once: true
|
||||
ignore_errors: yes
|
||||
|
||||
- name: Set backup fact
|
||||
ansible.builtin.set_fact:
|
||||
has_backups: "{{ (fallback_backup_check.rc == 0 and fallback_backup_check.stdout | trim != '') or (primary_backup_check.rc == 0 and primary_backup_check.stdout | trim != '') }}"
|
||||
when: inventory_hostname == 'pi1'
|
||||
run_once: true
|
||||
|
||||
- name: FAIL if no backups found
|
||||
ansible.builtin.fail:
|
||||
msg: "No backup files found in {{ primary_backup_dir }} or {{ fallback_backup_dir }}. Cannot proceed."
|
||||
when: inventory_hostname == 'pi1' and not has_backups | bool
|
||||
run_once: true
|
||||
|
||||
# ========================================================================
|
||||
# PHASE 1: Diagnosis - Check Current State
|
||||
# ========================================================================
|
||||
|
||||
- name: Gather Longhorn namespace status
|
||||
block:
|
||||
- name: Check if longhorn-system namespace exists
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Namespace
|
||||
name: "{{ longhorn_namespace }}"
|
||||
register: longhorn_ns
|
||||
ignore_errors: yes
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Check CSI driver registration
|
||||
kubernetes.core.k8s_info:
|
||||
kind: CSIDriver
|
||||
name: driver.longhorn.io
|
||||
register: csi_driver
|
||||
ignore_errors: yes
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Check Longhorn manager pods
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Pod
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
label_selectors:
|
||||
- app=longhorn-manager
|
||||
register: managers
|
||||
ignore_errors: yes
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Set recovery_phase fact
|
||||
ansible.builtin.set_fact:
|
||||
recovery_phase: "none"
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Determine recovery phase needed
|
||||
ansible.builtin.set_fact:
|
||||
recovery_phase: >-
|
||||
{% if csi_driver.failed %}
|
||||
soft
|
||||
{% elif managers.failed or managers.resources | default([]) | selectattr('status.phase', 'defined') | selectattr('status.phase', 'ne', 'Running') | list | length > 0 %}
|
||||
hard
|
||||
{% elif longhorn_ns.failed %}
|
||||
none
|
||||
{% else %}
|
||||
none
|
||||
{% endif %}
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Display recovery diagnosis
|
||||
ansible.builtin.debug:
|
||||
msg: "Diagnosis: recovery_phase={{ recovery_phase | default('none') }}. CSI Driver exists: {{ not csi_driver.failed | bool }}, Managers healthy: {{ managers.failed | ternary('unknown', managers.resources | default([]) | selectattr('status.phase', 'defined') | selectattr('status.phase', 'eq', 'Running') | list | length >= 3) | bool }}"
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
when: inventory_hostname == 'pi1'
|
||||
run_once: true
|
||||
|
||||
# ========================================================================
|
||||
# PHASE 2: Soft Recovery - Touch Manifest
|
||||
# ========================================================================
|
||||
|
||||
- name: Execute soft recovery - touch Longhorn manifest
|
||||
block:
|
||||
- name: Touch longhorn-install.yaml manifest
|
||||
ansible.builtin.file:
|
||||
path: "{{ longhorn_manifest_path }}"
|
||||
state: touch
|
||||
register: manifest_touch
|
||||
when: inventory_hostname == 'pi1'
|
||||
|
||||
- name: Wait for k3s to detect manifest change
|
||||
ansible.builtin.pause:
|
||||
minutes: 1
|
||||
when: manifest_touch is changed
|
||||
|
||||
- name: Check if Longhorn pods are recreating
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Pod
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
register: longhorn_pods
|
||||
ignore_errors: yes
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Verify soft recovery success
|
||||
ansible.builtin.set_fact:
|
||||
soft_recovery_success: >-
|
||||
{{ (longhorn_pods.resources | default([]) | selectattr('metadata.creationTimestamp', 'defined') | list | length) >= 10 }}
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
when: recovery_phase == 'soft' and inventory_hostname == 'pi1'
|
||||
run_once: true
|
||||
|
||||
# ========================================================================
|
||||
# PHASE 3: Hard Recovery - Delete Driver-Deployer
|
||||
# ========================================================================
|
||||
|
||||
- name: Execute hard recovery - delete driver-deployer pods
|
||||
block:
|
||||
- name: Get driver-deployer pods
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Pod
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
label_selectors:
|
||||
- app=longhorn-driver-deployer
|
||||
register: driver_deployer_pods
|
||||
ignore_errors: yes
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Delete driver-deployer pods
|
||||
kubernetes.core.k8s:
|
||||
state: absent
|
||||
kind: Pod
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
name: "{{ item.metadata.name }}"
|
||||
force: yes
|
||||
grace_period: 0
|
||||
loop: "{{ driver_deployer_pods.resources | default([]) }}"
|
||||
when: driver_deployer_pods.resources | default([]) | length > 0
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Wait for HelmChart to recreate driver-deployer
|
||||
ansible.builtin.pause:
|
||||
minutes: 2
|
||||
|
||||
- name: Check driver-deployer status
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Pod
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
label_selectors:
|
||||
- app=longhorn-driver-deployer
|
||||
register: new_driver_deployer
|
||||
ignore_errors: yes
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
when: (recovery_phase == 'hard' or (recovery_phase == 'soft' and not soft_recovery_success | default(false))) and inventory_hostname == 'pi1'
|
||||
run_once: true
|
||||
|
||||
# ========================================================================
|
||||
# PHASE 4: Nuclear Recovery - Full Reinstall
|
||||
# ========================================================================
|
||||
|
||||
- name: Execute nuclear recovery - full Longhorn reinstall
|
||||
block:
|
||||
# Step 1: Delete HelmChart
|
||||
- name: Delete Longhorn HelmChart
|
||||
kubernetes.core.k8s:
|
||||
state: absent
|
||||
kind: HelmChart
|
||||
namespace: "{{ longhorn_chart_namespace }}"
|
||||
name: "{{ longhorn_chart_name }}"
|
||||
force: yes
|
||||
grace_period: 0
|
||||
register: helmchart_deleted
|
||||
ignore_errors: yes
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Wait for HelmChart to be fully removed
|
||||
ansible.builtin.pause:
|
||||
seconds: 30
|
||||
when: helmchart_deleted is changed
|
||||
run_once: true
|
||||
|
||||
# Step 2: Remove Longhorn manifest from filesystem
|
||||
- name: Remove Longhorn manifest file
|
||||
ansible.builtin.file:
|
||||
path: "{{ longhorn_manifest_path }}"
|
||||
state: absent
|
||||
when: inventory_hostname == 'pi1'
|
||||
register: manifest_removed
|
||||
|
||||
# Step 3: Remove finalizers from all Longhorn resources
|
||||
- name: Get list of all Longhorn CRDs
|
||||
kubernetes.core.k8s_info:
|
||||
kind: CustomResourceDefinition
|
||||
label_selectors:
|
||||
- app=longhorn
|
||||
register: longhorn_crds
|
||||
ignore_errors: yes
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Get all Longhorn CR instances
|
||||
kubernetes.core.k8s_info:
|
||||
kind: "{{ item.spec.names.kind }}"
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
api_version: "{{ item.spec.group ~ '/' ~ item.spec.versions[0].name }}"
|
||||
register: cr_instances
|
||||
ignore_errors: yes
|
||||
loop: "{{ longhorn_crds.resources | default([]) }}"
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Remove finalizers from all Longhorn CR instances
|
||||
kubernetes.core.k8s_json_patch:
|
||||
kind: "{{ item.0.spec.names.kind }}"
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
name: "{{ item.1.metadata.name }}"
|
||||
api_version: "{{ item.0.spec.group ~ '/' ~ item.0.spec.versions[0].name }}"
|
||||
patch:
|
||||
- op: replace
|
||||
path: /metadata/finalizers
|
||||
value: []
|
||||
loop: >-
|
||||
{% set results = [] %}
|
||||
{% for crd in longhorn_crds.resources | default([]) %}
|
||||
{% for instance in hostvars['localhost']['cr_instances'].results | default([]) %}
|
||||
{% if instance.crd == crd %}
|
||||
{% set results = results.append([crd, instance.resources[0] if instance.resources else {}]) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
{{ results }}
|
||||
when: cr_instances.results | default([]) | length > 0
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
ignore_errors: yes
|
||||
|
||||
# Step 4: Remove finalizers from PVCs
|
||||
- name: Get all PVCs with longhorn storage class
|
||||
kubernetes.core.k8s_info:
|
||||
kind: PersistentVolumeClaim
|
||||
register: all_pvcs
|
||||
ignore_errors: yes
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Remove finalizers from PVCs
|
||||
kubernetes.core.k8s_json_patch:
|
||||
kind: PersistentVolumeClaim
|
||||
namespace: "{{ item.metadata.namespace }}"
|
||||
name: "{{ item.metadata.name }}"
|
||||
patch:
|
||||
- op: replace
|
||||
path: /metadata/finalizers
|
||||
value: []
|
||||
loop: "{{ all_pvcs.resources | default([]) | selectattr('spec.storageClassName', 'defined') | selectattr('spec.storageClassName', 'match', 'longhorn.*') | list }}"
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
ignore_errors: yes
|
||||
|
||||
# Step 5: Remove namespace finalizers
|
||||
- name: Remove finalizers from longhorn-system namespace
|
||||
kubernetes.core.k8s_json_patch:
|
||||
kind: Namespace
|
||||
name: "{{ longhorn_namespace }}"
|
||||
patch:
|
||||
- op: replace
|
||||
path: /spec/finalizers
|
||||
value: []
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
ignore_errors: yes
|
||||
|
||||
- name: Delete longhorn-system namespace
|
||||
kubernetes.core.k8s:
|
||||
state: absent
|
||||
kind: Namespace
|
||||
name: "{{ longhorn_namespace }}"
|
||||
force: yes
|
||||
grace_period: 0
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
ignore_errors: yes
|
||||
|
||||
- name: Wait for namespace deletion
|
||||
ansible.builtin.pause:
|
||||
seconds: 15
|
||||
run_once: true
|
||||
|
||||
# Step 6: Reinstall Longhorn via manifest
|
||||
- name: Deploy Longhorn HelmChart manifest
|
||||
ansible.builtin.copy:
|
||||
dest: "{{ longhorn_manifest_path }}"
|
||||
content: |
|
||||
apiVersion: helm.cattle.io/v1
|
||||
kind: HelmChart
|
||||
metadata:
|
||||
annotations:
|
||||
helmcharts.cattle.io/managed-by: helm-controller
|
||||
finalizers:
|
||||
- wrangler.cattle.io/on-helm-chart-remove
|
||||
name: longhorn-install
|
||||
namespace: kube-system
|
||||
spec:
|
||||
version: v1.9.1
|
||||
chart: longhorn
|
||||
repo: https://charts.longhorn.io
|
||||
failurePolicy: abort
|
||||
targetNamespace: longhorn-system
|
||||
createNamespace: true
|
||||
valuesContent: |-
|
||||
defaultSettings:
|
||||
defaultDataPath: {{ longhorn_data_path }}
|
||||
when: inventory_hostname == 'pi1'
|
||||
register: manifest_deployed
|
||||
|
||||
- name: Trigger k3s reconcile by touching manifest
|
||||
ansible.builtin.file:
|
||||
path: "{{ longhorn_manifest_path }}"
|
||||
state: touch
|
||||
when: manifest_deployed is changed and inventory_hostname == 'pi1'
|
||||
|
||||
- name: Wait for Longhorn pods to be created
|
||||
ansible.builtin.pause:
|
||||
minutes: 3
|
||||
when: manifest_deployed is changed
|
||||
run_once: true
|
||||
|
||||
when: >-
|
||||
(recovery_phase == 'hard' and not new_driver_deployer.resources | default([]) | selectattr('status.phase', 'eq', 'Running') | list | length > 0)
|
||||
or (recovery_phase == 'soft' and not soft_recovery_success | default(false) and not new_driver_deployer.resources | default([]) | selectattr('status.phase', 'eq', 'Running') | list | length > 0)
|
||||
or recovery_phase == 'none'
|
||||
run_once: true
|
||||
|
||||
# ========================================================================
|
||||
# PHASE 5: Restore from Backup
|
||||
# ========================================================================
|
||||
|
||||
- name: Execute restore from backup
|
||||
block:
|
||||
- name: Determine backup directory to use
|
||||
ansible.builtin.set_fact:
|
||||
backup_dir_to_use: >-
|
||||
{% if fallback_backup_dir and lookup('fileglob', fallback_backup_dir ~ '/backup_*.volumes') | length > 0 %}
|
||||
{{ fallback_backup_dir }}
|
||||
{% elif primary_backup_dir and lookup('fileglob', primary_backup_dir ~ '/backup_*.volumes') | length > 0 %}
|
||||
{{ primary_backup_dir }}
|
||||
{% else %}
|
||||
""
|
||||
{% endif %}
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: FAIL if no backup directory found
|
||||
ansible.builtin.fail:
|
||||
msg: "No valid backup directory found with backup_*.volumes files"
|
||||
when: backup_dir_to_use == ""
|
||||
run_once: true
|
||||
|
||||
- name: Find latest backup file
|
||||
ansible.builtin.set_fact:
|
||||
latest_backup: >-
|
||||
{% set files = lookup('fileglob', backup_dir_to_use ~ '/backup_*.volumes', wantlist=True) | sort(attribute='stat.mtime', reverse=True) %}
|
||||
{% if files | length > 0 %}
|
||||
{{ files[0].path }}
|
||||
{% endif %}
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: FAIL if no backup files found
|
||||
ansible.builtin.fail:
|
||||
msg: "No backup files found in {{ backup_dir_to_use }}"
|
||||
when: latest_backup | default('') == ''
|
||||
run_once: true
|
||||
|
||||
- name: Wait for Longhorn managers to be ready
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Pod
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
label_selectors:
|
||||
- app=longhorn-manager
|
||||
register: managers_status
|
||||
until: >-
|
||||
{{ (managers_status.resources | default([]) | selectattr('status.phase', 'eq', 'Running') | list | length) >= 1 }}
|
||||
retries: 30
|
||||
delay: 10
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Apply PV/PVC backup
|
||||
kubernetes.core.k8s:
|
||||
state: present
|
||||
src: "{{ latest_backup }}"
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Find Longhorn metadata backup
|
||||
ansible.builtin.set_fact:
|
||||
longhorn_backup: >-
|
||||
{% set lh_files = lookup('fileglob', backup_dir_to_use ~ '/longhorn_metadata_*.yaml', wantlist=True) | sort(attribute='stat.mtime', reverse=True) %}
|
||||
{% if lh_files | length > 0 %}
|
||||
{{ lh_files[0].path }}
|
||||
{% endif %}
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Apply Longhorn metadata backup (if exists)
|
||||
kubernetes.core.k8s:
|
||||
state: present
|
||||
src: "{{ longhorn_backup | default(omit) }}"
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
when: longhorn_backup | default('') != ''
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
when: inventory_hostname == 'pi1'
|
||||
run_once: true
|
||||
|
||||
# ========================================================================
|
||||
# PHASE 6: Post-Recovery Verification
|
||||
# ========================================================================
|
||||
|
||||
- name: Verify recovery success
|
||||
block:
|
||||
- name: Check CSI driver registration
|
||||
kubernetes.core.k8s_info:
|
||||
kind: CSIDriver
|
||||
name: driver.longhorn.io
|
||||
register: csi_final
|
||||
until: csi_final.resources | length > 0
|
||||
retries: 10
|
||||
delay: 10
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Check Longhorn manager health
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Pod
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
label_selectors:
|
||||
- app=longhorn-manager
|
||||
register: managers_final
|
||||
until: >-
|
||||
{{ (managers_final.resources | default([]) | selectattr('status.phase', 'eq', 'Running') | list | length) >= 3 }}
|
||||
retries: 15
|
||||
delay: 10
|
||||
run_once: true
|
||||
delegate_to: localhost
|
||||
|
||||
- name: Check CSI socket exists (on pi1)
|
||||
ansible.builtin.stat:
|
||||
path: /var/lib/kubelet/plugins/driver.longhorn.io/csi.sock
|
||||
register: csi_socket
|
||||
when: inventory_hostname == 'pi1'
|
||||
|
||||
- name: Verify volume data is still present
|
||||
ansible.builtin.stat:
|
||||
path: "{{ longhorn_data_path }}/replicas"
|
||||
register: replicas_dir
|
||||
when: inventory_hostname == 'pi1'
|
||||
|
||||
- name: Display recovery summary
|
||||
ansible.builtin.debug:
|
||||
msg: |
|
||||
===== Longhorn Recovery Summary =====
|
||||
CSI Driver Registered: {{ not csi_final.failed | bool | ternary('✓', '✗') }}
|
||||
Managers Running: {{ (managers_final.resources | default([]) | selectattr('status.phase', 'eq', 'Running') | list | length) }}/3
|
||||
CSI Socket Exists: {{ csi_socket.stat.exists | default(false) | bool | ternary('✓', '✗') }}
|
||||
Volume Data Present: {{ replicas_dir.stat.exists | default(false) | bool | ternary('✓', '✗') }}
|
||||
Backup Used: {{ latest_backup | default('none') }}
|
||||
======================================
|
||||
run_once: true
|
||||
|
||||
when: inventory_hostname == 'pi1'
|
||||
run_once: true
|
||||
@@ -0,0 +1,914 @@
|
||||
---
|
||||
# Longhorn Block-Device Data Recovery Playbook
|
||||
#
|
||||
# PURPOSE:
|
||||
# Recover application data directly from raw Longhorn replica files when Volume CRDs
|
||||
# are missing (e.g. after a nuclear cleanup + reinstall). Bypasses k8s objects entirely
|
||||
# and works at the block-device level.
|
||||
#
|
||||
# WHEN TO USE:
|
||||
# - Longhorn has been fully reinstalled (Volume CRDs are gone)
|
||||
# - Application PVCs are stuck Terminating / Lost
|
||||
# - The raw replica .img files still exist on disk
|
||||
# → See docs/runbooks/longhorn-block-device-recovery.md for the manual equivalent
|
||||
#
|
||||
# WHEN NOT TO USE:
|
||||
# - Volume CRDs still exist → use playbooks/recover/longhorn.yml instead
|
||||
# - All replica dirs were zeroed by Longhorn reconciliation (data is unrecoverable)
|
||||
#
|
||||
# USAGE:
|
||||
# ansible-playbook -i inventory/hosts.yml playbooks/recover/longhorn_data.yml \
|
||||
# -e @vars/recovery_volumes.yml
|
||||
#
|
||||
# VARS FILE FORMAT (vars/recovery_volumes.yml):
|
||||
# longhorn_recovery_volumes:
|
||||
# - pv_name: pvc-abc123 # Longhorn volume name (== PV name)
|
||||
# pvc_name: myapp-data # PVC name in the namespace
|
||||
# namespace: myapp # namespace where the PVC lives
|
||||
# size_bytes: "134217728" # volume size in bytes (string)
|
||||
# size_human: 128Mi # human-readable, used in PVC spec
|
||||
# access_mode: ReadWriteOnce # ReadWriteOnce or ReadWriteMany
|
||||
# workload_kind: Deployment # Deployment or StatefulSet
|
||||
# workload_name: myapp # name of the workload to scale down/up
|
||||
# source_node: pi3 # [OPTIONAL] node with untouched replica dir
|
||||
# source_dir: pvc-abc123-998f49ff # [OPTIONAL] exact replica dir name
|
||||
# verify_cmd: "" # optional: command to run inside pod to verify data after recovery
|
||||
#
|
||||
# source_node and source_dir are auto-discovered (largest dir >16K across all nodes)
|
||||
# when not specified. Override manually only to force a specific replica dir.
|
||||
#
|
||||
# REQUIREMENTS:
|
||||
# - python3 on all cluster nodes
|
||||
# - kubectl configured on the Ansible controller (localhost)
|
||||
# - longhorn-system namespace running and healthy before this playbook starts
|
||||
# - kubernetes.core collection: ansible-galaxy collection install kubernetes.core
|
||||
#
|
||||
# TESTED SCENARIO:
|
||||
# 2026-04-13 power cut — nuclear Longhorn reinstall — url-shortener SQLite recovery
|
||||
# Proven working as of 2026-04-14.
|
||||
|
||||
- name: Longhorn Block-Device Data Recovery
|
||||
hosts: localhost
|
||||
gather_facts: no
|
||||
|
||||
vars:
|
||||
longhorn_data_path: /mnt/arcodange/longhorn
|
||||
longhorn_namespace: longhorn-system
|
||||
longhorn_nodes: [pi1, pi2, pi3]
|
||||
merge_tool_local: "{{ playbook_dir }}/../../docs/incidents/2026-04-13-power-cut/tools/merge-longhorn-layers.py"
|
||||
merge_tool_remote: /home/pi/merge-longhorn-layers.py
|
||||
backup_base: /home/pi/arcodange/backups/longhorn-recovery
|
||||
merged_base: /tmp/longhorn-recovery-merged
|
||||
recovery_mount: /mnt/recovery-src
|
||||
live_mount: /mnt/recovery-live
|
||||
longhorn_recovery_volumes: [] # override with -e @vars/recovery_volumes.yml
|
||||
|
||||
tasks:
|
||||
|
||||
# =========================================================================
|
||||
# PRE-FLIGHT
|
||||
# =========================================================================
|
||||
|
||||
- name: "Pre-flight | Fail fast if no volumes defined"
|
||||
ansible.builtin.fail:
|
||||
msg: >
|
||||
No recovery volumes defined. Pass -e @vars/recovery_volumes.yml with a
|
||||
longhorn_recovery_volumes list. See playbook header for format.
|
||||
when: longhorn_recovery_volumes | length == 0
|
||||
|
||||
- name: "Pre-flight | Verify merge tool exists locally"
|
||||
ansible.builtin.stat:
|
||||
path: "{{ merge_tool_local }}"
|
||||
register: merge_tool_stat
|
||||
delegate_to: localhost
|
||||
|
||||
- name: "Pre-flight | Fail if merge tool missing"
|
||||
ansible.builtin.fail:
|
||||
msg: "merge-longhorn-layers.py not found at {{ merge_tool_local }}"
|
||||
when: not merge_tool_stat.stat.exists
|
||||
|
||||
- name: "Pre-flight | Check Longhorn is healthy"
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Pod
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
label_selectors:
|
||||
- app=longhorn-manager
|
||||
register: lh_managers
|
||||
delegate_to: localhost
|
||||
|
||||
- name: "Pre-flight | Fail if Longhorn managers are not running"
|
||||
ansible.builtin.fail:
|
||||
msg: >
|
||||
Longhorn managers not running (found {{ lh_managers.resources | default([]) |
|
||||
selectattr('status.phase', 'eq', 'Running') | list | length }} Running pods).
|
||||
Ensure Longhorn is healthy before attempting data recovery.
|
||||
when: >
|
||||
(lh_managers.resources | default([]) |
|
||||
selectattr('status.phase', 'eq', 'Running') | list | length) < 1
|
||||
|
||||
- name: "Pre-flight | Summary"
|
||||
ansible.builtin.debug:
|
||||
msg: >
|
||||
Longhorn healthy ({{ lh_managers.resources |
|
||||
selectattr('status.phase', 'eq', 'Running') | list | length }} managers running).
|
||||
Recovering {{ longhorn_recovery_volumes | length }} volume(s):
|
||||
{{ longhorn_recovery_volumes | map(attribute='pv_name') | list | join(', ') }}
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 0 — AUTO-DISCOVER BEST REPLICA DIR (when source_node/source_dir absent)
|
||||
# =========================================================================
|
||||
|
||||
- name: "Phase 0 | Scan replica dirs on all nodes"
|
||||
ansible.builtin.shell: |
|
||||
result=""
|
||||
for dir in {{ longhorn_data_path }}/replicas/{{ item.1.pv_name }}-*; do
|
||||
[ -d "$dir" ] || continue
|
||||
# Skip replicas that were being rebuilt — their data is incomplete
|
||||
meta="$dir/volume.meta"
|
||||
if [ -f "$meta" ]; then
|
||||
rebuilding=$(python3 -c "import json; d=json.load(open('$meta')); print(d.get('Rebuilding', False))" 2>/dev/null)
|
||||
[ "$rebuilding" = "True" ] && continue
|
||||
fi
|
||||
# Use actual disk usage (not apparent/sparse size) to rank replicas
|
||||
size=$(du -sk "$dir" 2>/dev/null | cut -f1)
|
||||
name=$(basename "$dir")
|
||||
result="$result\n$size $name"
|
||||
done
|
||||
printf '%b' "$result" | grep -v '^$' || true
|
||||
delegate_to: "{{ item.0 }}"
|
||||
become: yes
|
||||
loop: "{{ longhorn_nodes | product(longhorn_recovery_volumes) | list }}"
|
||||
loop_control:
|
||||
label: "{{ item.0 }}: {{ item.1.pv_name }}"
|
||||
register: dir_scan_raw
|
||||
changed_when: false
|
||||
when: item.1.source_node | default('') == '' or item.1.source_dir | default('') == ''
|
||||
|
||||
- name: "Phase 0 | Pick best source (largest dir with data, >16K)"
|
||||
ansible.builtin.set_fact:
|
||||
_discovered_sources: "{{ _build | from_json }}"
|
||||
vars:
|
||||
_build: >-
|
||||
{% set ns = namespace(result={}) %}
|
||||
{% for res in dir_scan_raw.results | default([]) %}
|
||||
{% if not res.skipped | default(false) and res.stdout | default('') != '' %}
|
||||
{% set node = res.item.0 %}
|
||||
{% set vol = res.item.1.pv_name %}
|
||||
{% for line in res.stdout_lines %}
|
||||
{% set parts = line.split() %}
|
||||
{% if parts | length == 2 %}
|
||||
{% set size = parts[0] | int %}
|
||||
{% set dir = parts[1] %}
|
||||
{% if size > 16384 and (vol not in ns.result or size > ns.result[vol].size) %}
|
||||
{# size is in KB (from du -sk); 16384 KB = 16 MiB minimum real replica #}
|
||||
{% set _ = ns.result.update({vol: {'node': node, 'dir': dir, 'size': size}}) %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{{ ns.result | to_json }}
|
||||
|
||||
- name: "Phase 0 | Show discovered sources"
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
{% for vol in longhorn_recovery_volumes %}
|
||||
{{ vol.pv_name }}:
|
||||
{% if vol.source_node | default('') != '' %}
|
||||
source: MANUAL → {{ vol.source_node }}/{{ vol.source_dir }}
|
||||
{% elif vol.pv_name in _discovered_sources %}
|
||||
source: AUTO → {{ _discovered_sources[vol.pv_name].node }}/{{ _discovered_sources[vol.pv_name].dir }}
|
||||
({{ (_discovered_sources[vol.pv_name].size / 1024 / 1024) | round(0) | int }} MiB)
|
||||
{% else %}
|
||||
source: NOT FOUND — no dir >16K on any node for this volume
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
- name: "Phase 0 | Fail if source not found for any volume"
|
||||
ansible.builtin.fail:
|
||||
msg: >
|
||||
No replica dir with data found for {{ item.pv_name }} on any node
|
||||
({{ longhorn_nodes | join(', ') }}). Check that the replica files survived.
|
||||
loop: "{{ longhorn_recovery_volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
when: >
|
||||
item.source_node | default('') == '' and
|
||||
item.source_dir | default('') == '' and
|
||||
item.pv_name not in _discovered_sources
|
||||
|
||||
- name: "Phase 0 | Initialize merged volume list"
|
||||
ansible.builtin.set_fact:
|
||||
_merged_volumes: []
|
||||
|
||||
- name: "Phase 0 | Append each volume with resolved source"
|
||||
ansible.builtin.set_fact:
|
||||
_merged_volumes: "{{ _merged_volumes + [item | combine(_source)] }}"
|
||||
vars:
|
||||
_manual: "{{ item.source_node | default('') != '' and item.source_dir | default('') != '' }}"
|
||||
_source: "{{ _manual | bool | ternary(
|
||||
{'source_node': item.source_node, 'source_dir': item.source_dir},
|
||||
{'source_node': _discovered_sources[item.pv_name].node,
|
||||
'source_dir': _discovered_sources[item.pv_name].dir}) }}"
|
||||
loop: "{{ longhorn_recovery_volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
|
||||
- name: "Phase 0 | Apply resolved volume list"
|
||||
ansible.builtin.set_fact:
|
||||
_volumes: "{{ _merged_volumes }}"
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 1 — UPLOAD MERGE TOOL AND BACK UP REPLICA DIRS
|
||||
# =========================================================================
|
||||
|
||||
- name: "Phase 1 | Upload merge tool to source nodes"
|
||||
ansible.builtin.command: >
|
||||
scp -o StrictHostKeyChecking=no
|
||||
{{ merge_tool_local }}
|
||||
pi@{{ item.source_node }}.home:{{ merge_tool_remote }}
|
||||
delegate_to: localhost
|
||||
become: no
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }} → {{ item.source_node }}"
|
||||
changed_when: true
|
||||
|
||||
- name: "Phase 1 | Create backup directory on source node"
|
||||
ansible.builtin.file:
|
||||
path: "{{ backup_base }}/{{ item.pvc_name }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
delegate_to: "{{ item.source_node }}"
|
||||
become: yes
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pvc_name }}"
|
||||
|
||||
- name: "Phase 1 | Check if backup already exists (skip if re-running)"
|
||||
ansible.builtin.stat:
|
||||
path: "{{ backup_base }}/{{ item.pvc_name }}/{{ item.source_dir }}/volume.meta"
|
||||
register: backup_exists
|
||||
delegate_to: "{{ item.source_node }}"
|
||||
become: yes
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pvc_name }}"
|
||||
|
||||
- name: "Phase 1 | Back up untouched replica dir (safe copy before any operation)"
|
||||
ansible.builtin.shell: >
|
||||
cp -a {{ longhorn_data_path }}/replicas/{{ item.item.source_dir }}
|
||||
{{ backup_base }}/{{ item.item.pvc_name }}/
|
||||
delegate_to: "{{ item.item.source_node }}"
|
||||
become: yes
|
||||
loop: "{{ backup_exists.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.pvc_name }}"
|
||||
when: not item.stat.exists
|
||||
changed_when: true
|
||||
|
||||
- name: "Phase 1 | Verify backup contains volume.meta"
|
||||
ansible.builtin.stat:
|
||||
path: "{{ backup_base }}/{{ item.pvc_name }}/{{ item.source_dir }}/volume.meta"
|
||||
register: backup_meta
|
||||
delegate_to: "{{ item.source_node }}"
|
||||
become: yes
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pvc_name }}"
|
||||
|
||||
- name: "Phase 1 | Fail if backup is incomplete"
|
||||
ansible.builtin.fail:
|
||||
msg: >
|
||||
Backup for {{ item.item.pvc_name }} is missing volume.meta — the source dir
|
||||
{{ item.item.source_dir }} may not exist or backup copy failed.
|
||||
loop: "{{ backup_meta.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.pvc_name }}"
|
||||
when: not item.stat.exists
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 2 — RECONSTRUCT FILESYSTEMS FROM REPLICA LAYERS
|
||||
# =========================================================================
|
||||
|
||||
- name: "Phase 2 | Create merged output directory"
|
||||
ansible.builtin.file:
|
||||
path: "{{ merged_base }}"
|
||||
state: directory
|
||||
mode: "0755"
|
||||
delegate_to: "{{ item.source_node }}"
|
||||
become: yes
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pvc_name }}"
|
||||
|
||||
- name: "Phase 2 | Check if merged image already exists"
|
||||
ansible.builtin.stat:
|
||||
path: "{{ merged_base }}/{{ item.pvc_name }}.img"
|
||||
register: merged_exists
|
||||
delegate_to: "{{ item.source_node }}"
|
||||
become: yes
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pvc_name }}"
|
||||
|
||||
- name: "Phase 2 | Merge snapshot + head layers into single image"
|
||||
ansible.builtin.command: >
|
||||
python3 {{ merge_tool_remote }}
|
||||
{{ backup_base }}/{{ item.item.pvc_name }}/{{ item.item.source_dir }}
|
||||
{{ merged_base }}/{{ item.item.pvc_name }}.img
|
||||
delegate_to: "{{ item.item.source_node }}"
|
||||
become: yes
|
||||
loop: "{{ merged_exists.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.pvc_name }}"
|
||||
when: not item.stat.exists
|
||||
changed_when: true
|
||||
register: merge_output
|
||||
|
||||
- name: "Phase 2 | Show merge output"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ item.stdout_lines | default([]) }}"
|
||||
loop: "{{ merge_output.results | default([]) }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.item.pvc_name | default('') }}"
|
||||
when: item.stdout_lines is defined
|
||||
|
||||
- name: "Phase 2 | Test mount merged image to verify filesystem"
|
||||
ansible.builtin.shell: |
|
||||
mkdir -p {{ recovery_mount }}-{{ item.pvc_name }}
|
||||
mount -o loop,ro,noload {{ merged_base }}/{{ item.pvc_name }}.img {{ recovery_mount }}-{{ item.pvc_name }}
|
||||
ls {{ recovery_mount }}-{{ item.pvc_name }}/
|
||||
umount {{ recovery_mount }}-{{ item.pvc_name }}
|
||||
delegate_to: "{{ item.source_node }}"
|
||||
become: yes
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pvc_name }}"
|
||||
register: mount_test
|
||||
changed_when: false
|
||||
|
||||
- name: "Phase 2 | Show filesystem contents"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ item.item.pvc_name }}: {{ item.stdout_lines }}"
|
||||
loop: "{{ mount_test.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.pvc_name }}"
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 3 — CREATE LONGHORN VOLUME CRDs
|
||||
# =========================================================================
|
||||
|
||||
# Scale down StatefulSets BEFORE removing PVC finalizers.
|
||||
# StatefulSet controllers auto-recreate PVCs as soon as they are deleted; if we
|
||||
# remove finalizers while the StatefulSet is still running, the controller
|
||||
# immediately provisions a new empty PVC (bound to a fresh volume), making the
|
||||
# PVC spec immutable by the time Phase 8 tries to pin it to our recovered PV.
|
||||
# Deployments are less urgent here but scaled early for consistency.
|
||||
|
||||
- name: "Phase 3 | Pre-scale down Deployments (before PVC finalizer removal)"
|
||||
kubernetes.core.k8s_scale:
|
||||
kind: Deployment
|
||||
name: "{{ item.workload_name }}"
|
||||
namespace: "{{ item.namespace }}"
|
||||
replicas: 0
|
||||
wait: yes
|
||||
wait_timeout: 60
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.workload_name }}"
|
||||
when: item.workload_kind == 'Deployment' and item.workload_name != ''
|
||||
ignore_errors: yes
|
||||
|
||||
- name: "Phase 3 | Pre-scale down StatefulSets (before PVC finalizer removal)"
|
||||
kubernetes.core.k8s_scale:
|
||||
kind: StatefulSet
|
||||
name: "{{ item.workload_name }}"
|
||||
namespace: "{{ item.namespace }}"
|
||||
replicas: 0
|
||||
wait: yes
|
||||
wait_timeout: 60
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.workload_name }}"
|
||||
when: item.workload_kind == 'StatefulSet' and item.workload_name != ''
|
||||
ignore_errors: yes
|
||||
|
||||
# Clear any stuck Terminating PVs/PVCs BEFORE creating Volume CRDs.
|
||||
# If old Terminating PVCs still exist when we create the Volume CRD, Longhorn
|
||||
# associates them and deletes the Volume CRD when the PVC finishes terminating.
|
||||
|
||||
- name: "Phase 3 | Check PVC state before touching finalizers"
|
||||
ansible.builtin.shell: >
|
||||
kubectl get pvc {{ item.pvc_name }} -n {{ item.namespace }}
|
||||
-o jsonpath='{.metadata.deletionTimestamp}' 2>/dev/null || true
|
||||
register: pvc_deletion_ts
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.pvc_name }}"
|
||||
changed_when: false
|
||||
|
||||
- name: "Phase 3 | Remove finalizers from stuck PV (if Terminating)"
|
||||
ansible.builtin.shell: >
|
||||
kubectl patch pv {{ item.pv_name }} --type=merge
|
||||
-p '{"metadata":{"finalizers":null}}' 2>/dev/null || true
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
changed_when: false
|
||||
|
||||
- name: "Phase 3 | Remove finalizers from stuck PVC (if Terminating)"
|
||||
ansible.builtin.shell: >
|
||||
kubectl patch pvc {{ item.pvc_name }} -n {{ item.namespace }}
|
||||
--type=merge -p '{"metadata":{"finalizers":null}}' 2>/dev/null || true
|
||||
delegate_to: localhost
|
||||
loop: "{{ pvc_deletion_ts.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.namespace }}/{{ item.item.pvc_name }}"
|
||||
when: item.stdout != ''
|
||||
changed_when: false
|
||||
|
||||
- name: "Phase 3 | Wait for stuck PVCs to fully delete before creating Volume CRDs"
|
||||
kubernetes.core.k8s_info:
|
||||
kind: PersistentVolumeClaim
|
||||
name: "{{ item.item.pvc_name }}"
|
||||
namespace: "{{ item.item.namespace }}"
|
||||
register: pvc_pre_check
|
||||
until: pvc_pre_check.resources | default([]) | length == 0
|
||||
retries: 12
|
||||
delay: 5
|
||||
delegate_to: localhost
|
||||
loop: "{{ pvc_deletion_ts.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.namespace }}/{{ item.item.pvc_name }}"
|
||||
when: item.stdout != ''
|
||||
|
||||
- name: "Phase 3 | Check if Longhorn Volume CRD already exists"
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Volume
|
||||
api_version: longhorn.io/v1beta2
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
name: "{{ item.pv_name }}"
|
||||
register: volume_crd_check
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
|
||||
- name: "Phase 3 | Create Longhorn Volume CRD"
|
||||
kubernetes.core.k8s:
|
||||
state: present
|
||||
definition:
|
||||
apiVersion: longhorn.io/v1beta2
|
||||
kind: Volume
|
||||
metadata:
|
||||
name: "{{ item.item.pv_name }}"
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
spec:
|
||||
accessMode: "{{ item.item.access_mode | lower | replace('readwriteonce', 'rwo') | replace('readwritemany', 'rwx') }}"
|
||||
dataEngine: v1
|
||||
frontend: blockdev
|
||||
numberOfReplicas: 3
|
||||
size: "{{ item.item.size_bytes }}"
|
||||
delegate_to: localhost
|
||||
loop: "{{ volume_crd_check.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.pv_name }}"
|
||||
when: item.resources | default([]) | length == 0
|
||||
|
||||
- name: "Phase 3 | Wait for Longhorn replicas to appear (stopped state)"
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Replica
|
||||
api_version: longhorn.io/v1beta2
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
label_selectors:
|
||||
- "longhornvolume={{ item.pv_name }}"
|
||||
register: replicas_check
|
||||
until: replicas_check.resources | default([]) | length >= 1
|
||||
retries: 24
|
||||
delay: 5
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
|
||||
- name: "Phase 3 | Wait for Volume status to be populated (webhook cache)"
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Volume
|
||||
api_version: longhorn.io/v1beta2
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
name: "{{ item.pv_name }}"
|
||||
register: vol_ready
|
||||
until: >
|
||||
(vol_ready.resources | default([]) | first | default({}) ).status.state | default('') != ''
|
||||
retries: 24
|
||||
delay: 5
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 4 — SCALE DOWN WORKLOADS
|
||||
# =========================================================================
|
||||
|
||||
- name: "Phase 4 | Scale down Deployments"
|
||||
kubernetes.core.k8s_scale:
|
||||
kind: Deployment
|
||||
name: "{{ item.workload_name }}"
|
||||
namespace: "{{ item.namespace }}"
|
||||
replicas: 0
|
||||
wait: yes
|
||||
wait_timeout: 60
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.workload_name }}"
|
||||
when: item.workload_kind == 'Deployment' and item.workload_name != ''
|
||||
ignore_errors: yes
|
||||
|
||||
- name: "Phase 4 | Scale down StatefulSets"
|
||||
kubernetes.core.k8s_scale:
|
||||
kind: StatefulSet
|
||||
name: "{{ item.workload_name }}"
|
||||
namespace: "{{ item.namespace }}"
|
||||
replicas: 0
|
||||
wait: yes
|
||||
wait_timeout: 60
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.workload_name }}"
|
||||
when: item.workload_kind == 'StatefulSet' and item.workload_name != ''
|
||||
ignore_errors: yes
|
||||
|
||||
- name: "Phase 4 | Delete any lingering Error-state pods that may hold volume attachments"
|
||||
ansible.builtin.shell: |
|
||||
kubectl get pods -n {{ item.namespace }} \
|
||||
--field-selector='status.phase=Failed' -o name | xargs -r kubectl delete -n {{ item.namespace }}
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}"
|
||||
changed_when: false
|
||||
ignore_errors: yes
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 5 — ATTACH VOLUME VIA MAINTENANCE TICKET
|
||||
# =========================================================================
|
||||
|
||||
- name: "Phase 5 | Create VolumeAttachment maintenance ticket"
|
||||
kubernetes.core.k8s:
|
||||
state: present
|
||||
definition:
|
||||
apiVersion: longhorn.io/v1beta2
|
||||
kind: VolumeAttachment
|
||||
metadata:
|
||||
name: "{{ item.pv_name }}"
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
spec:
|
||||
attachmentTickets:
|
||||
recovery:
|
||||
generation: 0
|
||||
id: recovery
|
||||
nodeID: "{{ item.source_node }}"
|
||||
parameters:
|
||||
disableFrontend: "false"
|
||||
type: longhorn-api
|
||||
volume: "{{ item.pv_name }}"
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }} → {{ item.source_node }}"
|
||||
|
||||
- name: "Phase 5 | Wait for volume to reach attached state"
|
||||
kubernetes.core.k8s_info:
|
||||
kind: Volume
|
||||
api_version: longhorn.io/v1beta2
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
name: "{{ item.pv_name }}"
|
||||
register: vol_state
|
||||
until: >
|
||||
(vol_state.resources | default([]) | first | default({}) ).status.state | default('') == 'attached'
|
||||
retries: 24
|
||||
delay: 5
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
|
||||
- name: "Phase 5 | Verify block device exists on target node"
|
||||
ansible.builtin.stat:
|
||||
path: "/dev/longhorn/{{ item.pv_name }}"
|
||||
register: blockdev_check
|
||||
delegate_to: "{{ item.source_node }}"
|
||||
become: yes
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
|
||||
- name: "Phase 5 | Fail if block device not present"
|
||||
ansible.builtin.fail:
|
||||
msg: >
|
||||
Block device /dev/longhorn/{{ item.item.pv_name }} not found on
|
||||
{{ item.item.source_node }} after volume attached — check Longhorn logs.
|
||||
loop: "{{ blockdev_check.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.pv_name }}"
|
||||
when: not item.stat.exists
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 6 — INJECT DATA INTO LIVE BLOCK DEVICE
|
||||
# =========================================================================
|
||||
|
||||
- name: "Phase 6 | Inject data via block device (mount, rsync, umount)"
|
||||
ansible.builtin.shell: |
|
||||
LIVE="{{ live_mount }}-{{ item.pvc_name }}"
|
||||
SRC="{{ recovery_mount }}-{{ item.pvc_name }}"
|
||||
BLOCKDEV="/dev/longhorn/{{ item.pv_name }}"
|
||||
MERGED="{{ merged_base }}/{{ item.pvc_name }}.img"
|
||||
|
||||
# Always unmount on exit (success or partial failure)
|
||||
cleanup() {
|
||||
mountpoint -q "$SRC" && umount "$SRC" || true
|
||||
mountpoint -q "$LIVE" && umount "$LIVE" || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir -p "$LIVE" "$SRC"
|
||||
|
||||
# Format if not already formatted (idempotent — safe on re-run)
|
||||
if ! blkid "$BLOCKDEV" | grep -q 'TYPE='; then
|
||||
mkfs.ext4 -F "$BLOCKDEV"
|
||||
fi
|
||||
|
||||
# Mount live block device if not already mounted
|
||||
if ! mountpoint -q "$LIVE"; then
|
||||
mount "$BLOCKDEV" "$LIVE"
|
||||
fi
|
||||
|
||||
# Mount merged recovery image read-only if not already mounted
|
||||
if ! mountpoint -q "$SRC"; then
|
||||
mount -o loop,ro,noload "$MERGED" "$SRC"
|
||||
fi
|
||||
|
||||
# Sync data — exclude lost+found
|
||||
# --ignore-errors: continue past unreadable files (e.g. corrupted parts from power cut)
|
||||
# rc=23 (partial transfer) is treated as success — bulk data transferred
|
||||
rsync -av --ignore-errors --exclude='lost+found' "$SRC/" "$LIVE/" || \
|
||||
{ RC=$?; [ $RC -eq 23 ] && echo "WARNING: rsync rc=23 (some files unreadable in source — expected for power-cut partitions)" || exit $RC; }
|
||||
delegate_to: "{{ item.source_node }}"
|
||||
become: yes
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pvc_name }}"
|
||||
register: inject_output
|
||||
changed_when: true
|
||||
|
||||
- name: "Phase 6 | Show rsync output"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ item.stdout_lines | default([]) }}"
|
||||
loop: "{{ inject_output.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.pvc_name }}"
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 7 — DETACH VOLUME
|
||||
# =========================================================================
|
||||
|
||||
- name: "Phase 7 | Remove recovery attachment ticket"
|
||||
kubernetes.core.k8s_json_patch:
|
||||
kind: VolumeAttachment
|
||||
api_version: longhorn.io/v1beta2
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
name: "{{ item.pv_name }}"
|
||||
patch:
|
||||
- op: remove
|
||||
path: /spec/attachmentTickets/recovery
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
ignore_errors: yes
|
||||
|
||||
- name: "Phase 7 | Wait for recovery ticket to be gone"
|
||||
kubernetes.core.k8s_info:
|
||||
kind: VolumeAttachment
|
||||
api_version: longhorn.io/v1beta2
|
||||
namespace: "{{ longhorn_namespace }}"
|
||||
name: "{{ item.pv_name }}"
|
||||
register: va_state
|
||||
until: >
|
||||
(va_state.resources | default([]) | first | default({}) ).spec.attachmentTickets.recovery is not defined
|
||||
retries: 24
|
||||
delay: 5
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 8 — RESTORE PV AND PVC
|
||||
# =========================================================================
|
||||
|
||||
- name: "Phase 8 | Create PersistentVolume (Retain, no claimRef)"
|
||||
kubernetes.core.k8s:
|
||||
state: present
|
||||
definition:
|
||||
apiVersion: v1
|
||||
kind: PersistentVolume
|
||||
metadata:
|
||||
name: "{{ item.pv_name }}"
|
||||
annotations:
|
||||
pv.kubernetes.io/provisioned-by: driver.longhorn.io
|
||||
spec:
|
||||
accessModes:
|
||||
- "{{ item.access_mode }}"
|
||||
capacity:
|
||||
storage: "{{ item.size_human }}"
|
||||
csi:
|
||||
driver: driver.longhorn.io
|
||||
fsType: ext4
|
||||
volumeHandle: "{{ item.pv_name }}"
|
||||
volumeAttributes:
|
||||
dataEngine: v1
|
||||
dataLocality: disabled
|
||||
disableRevisionCounter: "true"
|
||||
numberOfReplicas: "3"
|
||||
staleReplicaTimeout: "30"
|
||||
persistentVolumeReclaimPolicy: Retain
|
||||
storageClassName: longhorn
|
||||
volumeMode: Filesystem
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
|
||||
- name: "Phase 8 | Wait for PV to be Available or Bound"
|
||||
kubernetes.core.k8s_info:
|
||||
kind: PersistentVolume
|
||||
name: "{{ item.pv_name }}"
|
||||
register: pv_state
|
||||
until: >
|
||||
(pv_state.resources | default([]) | first | default({}) ).status.phase | default('')
|
||||
in ['Available', 'Bound']
|
||||
retries: 12
|
||||
delay: 5
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.pv_name }}"
|
||||
|
||||
- name: "Phase 8 | Check if PVC already bound to correct PV"
|
||||
ansible.builtin.shell: >
|
||||
kubectl get pvc {{ item.pvc_name }} -n {{ item.namespace }}
|
||||
-o jsonpath='{.spec.volumeName}' 2>/dev/null || true
|
||||
register: pvc_current_volume
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.pvc_name }}"
|
||||
changed_when: false
|
||||
|
||||
- name: "Phase 8 | Create PersistentVolumeClaim pinned to PV"
|
||||
kubernetes.core.k8s:
|
||||
state: present
|
||||
definition:
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: "{{ item.item.pvc_name }}"
|
||||
namespace: "{{ item.item.namespace }}"
|
||||
spec:
|
||||
accessModes:
|
||||
- "{{ item.item.access_mode }}"
|
||||
resources:
|
||||
requests:
|
||||
storage: "{{ item.item.size_human }}"
|
||||
storageClassName: longhorn
|
||||
volumeMode: Filesystem
|
||||
volumeName: "{{ item.item.pv_name }}"
|
||||
delegate_to: localhost
|
||||
loop: "{{ pvc_current_volume.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.namespace }}/{{ item.item.pvc_name }}"
|
||||
when: item.stdout != item.item.pv_name
|
||||
|
||||
- name: "Phase 8 | Wait for PVC to be Bound"
|
||||
kubernetes.core.k8s_info:
|
||||
kind: PersistentVolumeClaim
|
||||
namespace: "{{ item.namespace }}"
|
||||
name: "{{ item.pvc_name }}"
|
||||
register: pvc_state
|
||||
until: >
|
||||
(pvc_state.resources | default([]) | first | default({}) ).status.phase | default('') == 'Bound'
|
||||
retries: 12
|
||||
delay: 5
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.pvc_name }}"
|
||||
|
||||
# =========================================================================
|
||||
# PHASE 9 — SCALE UP AND VERIFY
|
||||
# =========================================================================
|
||||
|
||||
- name: "Phase 9 | Scale up Deployments"
|
||||
kubernetes.core.k8s_scale:
|
||||
kind: Deployment
|
||||
name: "{{ item.workload_name }}"
|
||||
namespace: "{{ item.namespace }}"
|
||||
replicas: 1
|
||||
wait: yes
|
||||
wait_timeout: 120
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.workload_name }}"
|
||||
when: item.workload_kind == 'Deployment' and item.workload_name != ''
|
||||
ignore_errors: yes
|
||||
|
||||
- name: "Phase 9 | Scale up StatefulSets"
|
||||
kubernetes.core.k8s_scale:
|
||||
kind: StatefulSet
|
||||
name: "{{ item.workload_name }}"
|
||||
namespace: "{{ item.namespace }}"
|
||||
replicas: 1
|
||||
wait: yes
|
||||
wait_timeout: 120
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.workload_name }}"
|
||||
when: item.workload_kind == 'StatefulSet' and item.workload_name != ''
|
||||
ignore_errors: yes
|
||||
|
||||
- name: "Phase 9 | Wait for workload to report ready replicas"
|
||||
kubernetes.core.k8s_info:
|
||||
kind: "{{ item.workload_kind }}"
|
||||
name: "{{ item.workload_name }}"
|
||||
namespace: "{{ item.namespace }}"
|
||||
register: workload_state
|
||||
until: >
|
||||
(workload_state.resources | default([]) | first | default({}) ).status.readyReplicas | default(0) | int >= 1
|
||||
retries: 24
|
||||
delay: 5
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.workload_name }}"
|
||||
when: item.workload_name != ''
|
||||
ignore_errors: yes
|
||||
|
||||
- name: "Phase 9 | Run optional verification command in pod"
|
||||
ansible.builtin.shell: >
|
||||
kubectl exec -n {{ item.namespace }}
|
||||
$(kubectl get pod -n {{ item.namespace }}
|
||||
-l statefulset.kubernetes.io/pod-name={{ item.workload_name }}-0
|
||||
--no-headers -o custom-columns=':metadata.name' 2>/dev/null ||
|
||||
kubectl get pod -n {{ item.namespace }} {{ item.workload_name }}-0
|
||||
--no-headers -o custom-columns=':metadata.name' 2>/dev/null)
|
||||
-- sh -c '{{ item.verify_cmd }}'
|
||||
delegate_to: localhost
|
||||
loop: "{{ _volumes }}"
|
||||
loop_control:
|
||||
label: "{{ item.namespace }}/{{ item.workload_name }}"
|
||||
when: item.verify_cmd | default('') != ''
|
||||
register: verify_output
|
||||
changed_when: false
|
||||
ignore_errors: yes
|
||||
|
||||
- name: "Phase 9 | Show verification output"
|
||||
ansible.builtin.debug:
|
||||
msg: "{{ item.stdout_lines | default([]) }}"
|
||||
loop: "{{ verify_output.results | default([]) }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.pvc_name | default('') }}"
|
||||
when: item.stdout_lines is defined and item.item.verify_cmd | default('') != ''
|
||||
|
||||
# =========================================================================
|
||||
# RECOVERY SUMMARY
|
||||
# =========================================================================
|
||||
|
||||
- name: "Summary | Recovery complete"
|
||||
ansible.builtin.debug:
|
||||
msg: |
|
||||
╔══════════════════════════════════════════════════════╗
|
||||
║ Longhorn Block-Device Recovery Complete ║
|
||||
╚══════════════════════════════════════════════════════╝
|
||||
Volumes recovered:
|
||||
{% for v in _volumes %}
|
||||
• {{ v.pvc_name }} ({{ v.namespace }}) ← {{ v.source_node }}:{{ v.source_dir }}
|
||||
{% endfor %}
|
||||
|
||||
Backups retained at: {{ backup_base }}/<pvc-name>/
|
||||
Merged images at: {{ merged_base }}/<pvc-name>.img
|
||||
|
||||
Next steps:
|
||||
1. Verify application data through the app UI / API
|
||||
2. Repeat for remaining volumes (update vars file)
|
||||
3. Run a fresh k8s_pvc backup once all volumes are healthy
|
||||
@@ -0,0 +1,84 @@
|
||||
---
|
||||
# Example vars file for playbooks/recover/longhorn_data.yml
|
||||
#
|
||||
# Usage:
|
||||
# ansible-playbook -i inventory/hosts.yml playbooks/recover/longhorn_data.yml \
|
||||
# -e @playbooks/recover/longhorn_data_vars.example.yml
|
||||
#
|
||||
# HOW TO FILL THIS IN:
|
||||
#
|
||||
# 1. Find untouched replica dirs across all nodes:
|
||||
# for node in pi1 pi2 pi3; do
|
||||
# echo "=== $node ==="
|
||||
# ssh $node "sudo du -sh /mnt/arcodange/longhorn/replicas/pvc-<VOLUME>-* 2>/dev/null"
|
||||
# done
|
||||
# Pick the dir with the largest size (>16K) and oldest timestamps (from before the incident).
|
||||
#
|
||||
# 2. Get pv_name and pvc_name from PV/PVC backup:
|
||||
# cat /home/pi/arcodange/backups/k3s_pvc/backup_*.volumes | grep -A5 "kind: PersistentVolume"
|
||||
#
|
||||
# 3. Get size_bytes from Longhorn volume spec or from:
|
||||
# cat /mnt/arcodange/longhorn/replicas/<source_dir>/volume.meta
|
||||
#
|
||||
# 4. source_node = the node where the untouched dir lives
|
||||
# source_dir = the exact directory name (e.g. pvc-abc123-998f49ff)
|
||||
#
|
||||
# Fields:
|
||||
# pv_name — Longhorn volume name, equals the PV name (pvc-<uuid>) [REQUIRED]
|
||||
# pvc_name — PVC name in the namespace [REQUIRED]
|
||||
# namespace — namespace where the PVC lives [REQUIRED]
|
||||
# size_bytes — volume capacity in bytes as a string (from volume spec) [REQUIRED]
|
||||
# size_human — human-readable size for PVC spec (e.g. 128Mi, 8Gi) [REQUIRED]
|
||||
# access_mode — ReadWriteOnce or ReadWriteMany [REQUIRED]
|
||||
# workload_kind — Deployment or StatefulSet [REQUIRED]
|
||||
# workload_name — name of the workload to scale down/up [REQUIRED]
|
||||
# source_node — node holding the untouched replica dir (pi1/pi2/pi3) [OPTIONAL — auto-discovered]
|
||||
# source_dir — exact replica dir name on source_node [OPTIONAL — auto-discovered]
|
||||
# verify_cmd — shell command to run inside pod to confirm data after restore [OPTIONAL]
|
||||
#
|
||||
# source_node and source_dir are auto-discovered by Phase 0 (largest dir >16K across all
|
||||
# nodes). Override them manually only if you want to force a specific replica dir.
|
||||
|
||||
longhorn_recovery_volumes:
|
||||
|
||||
# --- url-shortener (example, already recovered 2026-04-14) ---
|
||||
- pv_name: pvc-cdd434d1-c8b4-4a75-acde-2978ec9febd4
|
||||
pvc_name: url-shortener-data
|
||||
namespace: url-shortener
|
||||
size_bytes: "134217728"
|
||||
size_human: 128Mi
|
||||
access_mode: ReadWriteOnce
|
||||
workload_kind: Deployment
|
||||
workload_name: url-shortener
|
||||
source_node: pi3
|
||||
source_dir: pvc-cdd434d1-c8b4-4a75-acde-2978ec9febd4-998f49ff
|
||||
verify_cmd: "sqlite3 /data/urls.db 'SELECT COUNT(*) FROM urls;'"
|
||||
|
||||
# --- traefik (example, already recovered 2026-04-14) ---
|
||||
# - pv_name: pvc-<traefik-uuid>
|
||||
# pvc_name: traefik-data
|
||||
# namespace: traefik
|
||||
# size_bytes: "134217728"
|
||||
# size_human: 128Mi
|
||||
# access_mode: ReadWriteOnce
|
||||
# workload_kind: Deployment
|
||||
# workload_name: traefik
|
||||
# source_node: pi3
|
||||
# source_dir: pvc-<traefik-uuid>-<hex>
|
||||
# verify_cmd: ""
|
||||
|
||||
# --- vault (uncomment and fill for recovery) ---
|
||||
# - pv_name: pvc-<vault-uuid>
|
||||
# pvc_name: vault-data
|
||||
# namespace: vault
|
||||
# size_bytes: "1073741824"
|
||||
# size_human: 1Gi
|
||||
# access_mode: ReadWriteOnce
|
||||
# workload_kind: StatefulSet
|
||||
# workload_name: vault
|
||||
# source_node: pi2
|
||||
# source_dir: pvc-<vault-uuid>-<hex>
|
||||
# verify_cmd: ""
|
||||
|
||||
# Add more volumes here following the same pattern.
|
||||
# Process one at a time first to validate, then batch.
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
# Recovery vars for Clickhouse
|
||||
# Source: pi3, dir pvc-1251909b-...-1163420b (2.6G — largest, snapshot verified non-zero)
|
||||
# Generated: 2026-04-14
|
||||
|
||||
longhorn_recovery_volumes:
|
||||
- pv_name: pvc-1251909b-3cef-40c6-881c-3bb6e929a596
|
||||
pvc_name: clickhouse-storage-clickhouse-0
|
||||
namespace: tools
|
||||
size_bytes: "17179869184" # 16Gi
|
||||
size_human: 16Gi
|
||||
access_mode: ReadWriteOnce
|
||||
workload_kind: StatefulSet
|
||||
workload_name: clickhouse
|
||||
source_node: pi3
|
||||
source_dir: pvc-1251909b-3cef-40c6-881c-3bb6e929a596-1163420b
|
||||
verify_cmd: "clickhouse-client --query 'SHOW DATABASES'"
|
||||
@@ -0,0 +1,38 @@
|
||||
---
|
||||
# Recovery vars for erp and hashicorp-vault volumes
|
||||
# source_node/source_dir omitted — auto-discovered by Phase 0
|
||||
|
||||
longhorn_recovery_volumes:
|
||||
|
||||
- pv_name: pvc-7971918e-e47f-4739-a976-965ea2d770b4
|
||||
pvc_name: erp
|
||||
namespace: erp
|
||||
size_bytes: "53687091200"
|
||||
size_human: 50Gi
|
||||
access_mode: ReadWriteMany
|
||||
workload_kind: Deployment
|
||||
workload_name: "" # intentionally blank — ERP needs Vault unsealed first; scale up manually
|
||||
verify_cmd: ""
|
||||
|
||||
# hashicorp-vault StatefulSet has two PVCs (audit + data).
|
||||
# workload_name is set only on the last entry so the StatefulSet is scaled up
|
||||
# once after both volumes are ready, not between them.
|
||||
- pv_name: pvc-6d2ea1c7-9327-4992-a02c-93ae604eda70
|
||||
pvc_name: audit-hashicorp-vault-0
|
||||
namespace: tools
|
||||
size_bytes: "10737418240"
|
||||
size_human: 10Gi
|
||||
access_mode: ReadWriteOnce
|
||||
workload_kind: StatefulSet
|
||||
workload_name: ""
|
||||
verify_cmd: ""
|
||||
|
||||
- pv_name: pvc-ca5567d3-a682-4cee-8ff1-2b8e23260635
|
||||
pvc_name: data-hashicorp-vault-0
|
||||
namespace: tools
|
||||
size_bytes: "10737418240"
|
||||
size_human: 10Gi
|
||||
access_mode: ReadWriteOnce
|
||||
workload_kind: StatefulSet
|
||||
workload_name: hashicorp-vault
|
||||
verify_cmd: ""
|
||||
@@ -0,0 +1,47 @@
|
||||
---
|
||||
# Recovery vars for remaining volumes (prometheus, alertmanager, redis, backups-rwx)
|
||||
# source_node and source_dir intentionally omitted — auto-discovered by Phase 0
|
||||
|
||||
longhorn_recovery_volumes:
|
||||
|
||||
- pv_name: pvc-88e18c7f-2cfd-45e3-be5b-78c31ab829e9
|
||||
pvc_name: prometheus-server
|
||||
namespace: tools
|
||||
size_bytes: "8589934592"
|
||||
size_human: 8Gi
|
||||
access_mode: ReadWriteOnce
|
||||
workload_kind: Deployment
|
||||
workload_name: prometheus-server
|
||||
source_node: pi2
|
||||
source_dir: pvc-88e18c7f-2cfd-45e3-be5b-78c31ab829e9-910583f6
|
||||
verify_cmd: ""
|
||||
|
||||
- pv_name: pvc-aed7f2c4-1948-487a-8d10-d8a1372289b4
|
||||
pvc_name: storage-prometheus-alertmanager-0
|
||||
namespace: tools
|
||||
size_bytes: "2147483648"
|
||||
size_human: 2Gi
|
||||
access_mode: ReadWriteOnce
|
||||
workload_kind: StatefulSet
|
||||
workload_name: prometheus-alertmanager
|
||||
verify_cmd: ""
|
||||
|
||||
- pv_name: pvc-d1d5482b-81c8-4d7c-a528-7a57ef47a5ce
|
||||
pvc_name: redis-storage-redis-0
|
||||
namespace: tools
|
||||
size_bytes: "1073741824"
|
||||
size_human: 1Gi
|
||||
access_mode: ReadWriteOnce
|
||||
workload_kind: StatefulSet
|
||||
workload_name: redis
|
||||
verify_cmd: "redis-cli ping"
|
||||
|
||||
- pv_name: pvc-efda1d2f-1db8-46dd-9a97-3d11f1807ffa
|
||||
pvc_name: backups-rwx
|
||||
namespace: longhorn-system
|
||||
size_bytes: "53687091200"
|
||||
size_human: 50Gi
|
||||
access_mode: ReadWriteMany
|
||||
workload_kind: Deployment
|
||||
workload_name: ""
|
||||
verify_cmd: ""
|
||||
Reference in New Issue
Block a user