configure vault secrets operator

.gitea/workflows/vault.yaml

@@ -2,7 +2,11 @@
 # template source: https://github.com/bretfisher/docker-build-workflow/blob/main/templates/call-docker-build.yaml
 name: Hashicorp Vault
 
-on: [push,pull_request]
+on: #[push,pull_request]
+  push: &vaultPaths
+    paths:
+      - 'hashicorp-vault/**/*.tf'
+  pull_request: *vaultPaths
 
 # cancel any previously-started, yet still active runs of this workflow on the same branch
 concurrency:
@@ -21,6 +25,8 @@ concurrency:
     path: gitea_jwt
     secrets: |
         kvv1/google/credentials credentials | GOOGLE_BACKEND_CREDENTIALS ;
+        kvv1/gitea/app id | TF_VAR_gitea_app_id ;
+        kvv1/postgres/credentials_editor/credentials * | TF_VAR_POSTGRES_CREDENTIALS_EDITOR_ ;
 
 jobs:
   gitea_vault_auth:
@@ -34,44 +40,6 @@ jobs:
         id: gitea_vault_jwt
         run: |
           echo -n "${{ secrets.vault_oauth__sh_b64 }}" | base64 -d | bash
-          echo "### Hello world! :rocket:" >> $GITHUB_STEP_SUMMARY
-      # - name: Step following 'Import Secrets'
-      #   run: |
-      #     echo "${{ env.GOOGLE_BACKEND_CREDENTIALS }}" | base64 -d | jq
-      #     echo $GOOGLE_BACKEND_CREDENTIALS | base64 -d | jq
-      #     echo "${{ steps.vault-secrets.outputs.GOOGLE_BACKEND_CREDENTIALS }}" | base64 -d | jq
-
-      # - uses: actions/checkout@v4
-  
-  # validate:
-  #   runs-on: ubuntu-latest
-  #   name: Validate terraform configuration
-  #   env:
-  #     OPENTOFU_VERSION: 1.8.2
-  #     TERRAFORM_VAULT_AUTH_JWT: ${{ needs.gitea_vault_auth.outputs.access_token }}
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v4
-
-  #     - name: terraform validate
-  #       uses: dflook/terraform-validate@v1
-  #       with:
-  #         path: hashicorp-vault/iac
-
-  # fmt-check:
-  #   runs-on: ubuntu-latest
-  #   name: Check formatting of terraform files
-  #   env:
-  #     OPENTOFU_VERSION: 1.8.2
-  #     TERRAFORM_VAULT_AUTH_JWT: ${{ needs.gitea_vault_auth.outputs.access_token }}
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v4
-
-  #     - name: terraform fmt
-  #       uses: dflook/terraform-fmt-check@v1
-  #       with:
-  #         path: hashicorp-vault/iac
 
   tofu:
     name: Tofu - Vault IAC
@@ -84,23 +52,11 @@ jobs:
     steps:
       - *vault_step
       - uses: actions/checkout@v4
-      - uses: dflook/terraform-plan@v1
+      # - uses: dflook/terraform-plan@v1
-        with:
-          path: hashicorp-vault/iac
-      # - name: terraform apply
-      #   uses: dflook/terraform-apply@v1
       #   with:
       #     path: hashicorp-vault/iac
-
+      - name: terraform apply
-  # - name: init
+        uses: dflook/terraform-apply@v1
-  #   uses: docker://ghcr.io/opentofu/opentofu:latest
+        with:
-  #   run: |
+          path: hashicorp-vault/iac
-  #     tofu -chdir=hashicorp-vault/iac init
+          auto_approve: true
-  # - name: plan
-  #   uses: docker://ghcr.io/opentofu/opentofu:latest
-  #   run: |
-  #     tofu -chdir=hashicorp-vault/iac plan -no-color -lock=false
-  # - name: apply
-  #   uses: docker://ghcr.io/opentofu/opentofu:latest
-  #   run: |
-  #     tofu -chdir=hashicorp-vault/iac apply -no-color -auto-approve

chart/templates/project.yaml

@@ -23,3 +23,5 @@ spec:
       kind: ClusterRole
     - group: '*'
       kind: MutatingWebhookConfiguration
+    - group: 'apiextensions.k8s.io'
+      kind: CustomResourceDefinition

hashicorp-vault/.helmignore

@@ -22,3 +22,6 @@
 *.tmproj
 .vscode/
 iac/
+README.md
+.terraform
+.terraform.lock

hashicorp-vault/Chart.yaml

@@ -9,6 +9,9 @@ dependencies:
 - name: vault
   version: 0.28.1
   repository: https://helm.releases.hashicorp.com
+- name: vault-secrets-operator
+  version: 0.9.0
+  repository: https://helm.releases.hashicorp.com
 
 # A chart can be either an 'application' or a 'library' chart.
 #

hashicorp-vault/README.md

@@ -0,0 +1,151 @@
+# Vault
+
+1. Les [playbooks ansible](https://gitea.arcodange.duckdns.org/arcodange-org/factory/src/branch/main/ansible/arcodange/factory/playbooks) configurent la base de données postgres et le minimum requis pour permetre au dépot "tools" d'appliquer via un workflow gitea action [une configuration vault via tofu](./iac/).
+2. Configuration des backend d'authentification et des roles pour postgres et kubernetes. Définition de rôles "${app}-ops" pour permettre au dépot d'une application de définir ses propres dépendances dans vault. Rotation de credentials postgres pour les applications.
+3. [Le dépot de l'application webapp](https://gitea.arcodange.duckdns.org/arcodange-org/webapp) gère l'obtention de ses crédentials pour postgres.
+
+```mermaid
+flowchart LR
+
+    classDef playbook stroke:#0f0,fill:#440,stroke-width: 1px
+    classDef tofu stroke:#f00,fill:#404,stroke-width: 2px
+    classDef argocd stroke:#00f,fill:#044,stroke-width: 3px
+    classDef database stroke:#bb0,fill:#ff0,stroke-width: 5px,color: black
+    classDef secret stroke:#f00,fill:#f00,stroke-width: 5px,color: yellow
+    classDef secretOperator stroke:#f00,fill:DarkRed,stroke-width: 5px,color: Orange
+
+    subgraph git_code[factory.git]
+        subgraph ansible_collection
+            setup_playbook[playbook arcodange.factory.setup]:::playbook
+            tools_playbook[playbook arcodange.factory.tools]:::playbook
+        end
+        git_code_tofu_vault{{tofu}}:::tofu
+    end
+
+    subgraph git_tools[tools.git]
+        argocd_tools{{Argo CD Apps}}:::argocd
+        git_tools_tofu_vault{{tofu}}:::tofu
+    end
+
+    subgraph git_webapp[webapp.git]
+        webapp["Go(lang) web app"]
+        argocd_webapp{{Argo CD App}}:::argocd
+        git_webapp_tofu_vault{{tofu}}:::tofu
+    end
+
+    subgraph servers
+        subgraph k3s
+            subgraph k3s_ns_tools[ns:tools]
+                argocd{{Argo CD}}:::argocd
+                pgbouncer
+                subgraph vault
+                    subgraph vault_auth[auth]
+                        subgraph vault_auth_openid[openid]
+                        end
+                        vault_auth_jwt[jwt]
+                        vault_auth_k8s[kubernetes]
+                        vault_auth_jwt_role_gitea_cicd[gitea_cicd role]
+                        vault_auth_jwt_role_gitea_cicd_webapp_ops[gitea_cicd_webapp ops role]
+                        vault_auth_k8s_role_vso[vault-secret-operator role]
+                        vault_auth_k8s_role_webapp[webapp role]
+                        subgraph policies
+                            policy_default[default]
+                            policy_webapp[webapp]
+                            policy_webapp_ops[webapp ops]
+                            policy_admin[admin]
+                            policy_vso[edit-vso-client-cache]
+                        end
+                    end
+                    subgraph vault_secrets[secrets]
+                        subgraph kvv2
+                            google/credentials
+                            webapp/config
+                        end
+                    end
+                    subgraph vault_postgres[postgres]
+                        creds/creds-editor
+                        creds/webapp
+                    end
+                    subgraph vault_transit[transit]
+                    end
+                end
+                vault-secret-operator:::secretOperator
+            end
+            subgraph k3s_ns_webapp[ns:webapp]
+                webapp_deployment[deployment:webapp]
+                webapp_postgres_creds_secret[secret:postgres creds]:::secret
+                webapp_config_secret[secret:config]:::secret
+                webapp_service_account[sa:webapp]
+            end
+        end
+        subgraph postgres
+            root_credentials
+            postgres_db[(postgres)]:::database
+            webapp_credentials:::secret
+            webapp_db[(webpp)]:::database
+
+            vault_creds_editor_role{{credentials_editor}}
+        end
+    end
+
+    setup_playbook -. setup postgres .-> postgres
+    
+    tools_playbook -.-o git_code_tofu_vault
+    git_code_tofu_vault -..-> vault_auth_openid
+    git_code_tofu_vault -..-> vault_auth_jwt -- tofu:factory --- vault_auth_jwt_role_gitea_cicd
+    git_code_tofu_vault -..-> kvv2
+    git_code_tofu_vault -..-> google/credentials
+
+    linkStyle 0,1 stroke:#ff3,stroke-width:1px,color:DarkKhaki;
+    linkStyle 2,3,5,6 stroke:#f3f,stroke-width:2px,color:DarkOrange;
+    
+    git_tools -.-o argocd_tools
+    argocd_tools -.-> pgbouncer
+    argocd_tools -.-> vault
+    argocd_tools -.-> vault-secret-operator
+    argocd_tools o--o argocd
+
+    linkStyle 7,8,9,10,11 stroke:#3ff,stroke-width:3px,color:DarkSlateBlue;
+
+    git_tools_tofu_vault -..-> vault_auth_k8s -- sa:vso --- vault_auth_k8s_role_vso
+    git_tools_tofu_vault -..-> webapp/config
+    git_tools_tofu_vault -..-> vault_transit
+    git_tools_tofu_vault -..-> vault_postgres
+    vault_auth_k8s ---> k3s
+    vault_postgres --> pgbouncer x==> postgres; webapp_deployment --> pgbouncer
+
+    linkStyle 12,14,15,16 stroke:#f3f,stroke-width:2px,color:DarkOrange;
+    linkStyle 18,19,20 stroke:gold,stroke-width:2px;
+    
+    vault_transit x---x vault-secret-operator
+    vault-secret-operator x---x vault_auth_k8s_role_vso
+
+    vault_auth_jwt_role_gitea_cicd x--x policy_default
+    vault_auth_k8s_role_vso x--x policy_vso
+
+    creds/webapp -.-> webapp_credentials
+    creds/webapp -.-> vault-secret-operator
+    vault-secret-operator -.-> webapp_postgres_creds_secret
+    webapp/config -.-> vault-secret-operator
+    vault-secret-operator -.-> webapp_config_secret
+
+    argocd_webapp -.-> k3s_ns_webapp
+    webapp --o webapp_deployment
+    webapp_postgres_creds_secret --o webapp_deployment
+    webapp_deployment --> webapp_service_account
+
+    vault_auth_jwt -- tofu:tools --- vault_auth_jwt_role_gitea_cicd_webapp_ops
+    vault_auth_jwt_role_gitea_cicd_webapp_ops x--x policy_webapp_ops
+    vault_auth_k8s -- sa:webapp --- vault_auth_k8s_role_webapp x-- tofu:webapp --x policy_webapp
+
+    git_webapp_tofu_vault -.-> vault_auth_k8s_role_webapp
+    git_webapp_tofu_vault -.-> creds/webapp
+
+
+    root_credentials x--x postgres_db
+    webapp_credentials x--x webapp_db
+
+    tools_playbook --> vault_creds_editor_role
+    vault_creds_editor_role -. change password .-> webapp_credentials
+    vault_postgres x--x vault_creds_editor_role
+```

hashicorp-vault/iac/main.tf

@@ -5,11 +5,6 @@ terraform {
   }
 }
 
-variable "vault_address" {
-  type = string
-  default = "http://127.0.0.1:8200"
-}
-
 terraform {
     required_providers {
       vault = {
@@ -20,20 +15,97 @@ terraform {
 }
 
 provider vault {
-    address = var.vault_address
+    address = "https://vault.arcodange.duckdns.org"
     auth_login_jwt { # TERRAFORM_VAULT_AUTH_JWT environment variable
-        role = "admin"
+        mount = "gitea_jwt"
+        role = "gitea_cicd"
     }
 }
 
-data "vault_policy_document" "admin" {
+resource "vault_auth_backend" "kubernetes" {
+  type = "kubernetes"
+}
+resource "vault_kubernetes_auth_backend_config" "config" {
+  backend                = vault_auth_backend.kubernetes.path
+  kubernetes_host        = "https://kubernetes.default.svc:443"
+}
+
+resource "vault_mount" "kvv2" {
+  path        = "kvv2"
+  type        = "kv"
+  options     = { version = "2" }
+  description = "KV Version 2 secret engine mount"
+}
+
+
+resource "vault_mount" "postgres" {
+  path = "postgres"
+  type = "database"
+}
+
+resource "vault_database_secret_backend_connection" "postgres" {
+  backend       = vault_mount.postgres.path
+  name          = "postgres"
+  allowed_roles = ["*"]
+  root_rotation_statements = [
+    "ALTER USER \"{{name}}\" WITH PASSWORD '{{password}}';",
+  ]
+
+  postgresql {
+    connection_url = "postgresql://{{username}}:{{password}}@pgbouncer.tools:5432/postgres?sslmode=disable"
+    username = var.POSTGRES_CREDENTIALS_EDITOR_USERNAME
+    password = var.POSTGRES_CREDENTIALS_EDITOR_PASSWORD
+  }
+}
+
+
+resource "vault_mount" "transit" {
+  path                      = "transit"
+  type                      = "transit"
+  description               = "Pour le vault secret operator (vso) dans k3s en cas de redemarrage par exemple"
+#   default_lease_ttl_seconds = 3600
+#   max_lease_ttl_seconds     = 86400
+}
+resource "vault_transit_secret_backend_key" "vso_client_cache" {
+  backend = vault_mount.transit.path
+  name    = "vso-client-cache"
+}
+
+data "vault_policy_document" "vso_client_cache" {
   rule {
-    path         = "*"
+    path = "${vault_mount.transit.path}/encrypt/${vault_transit_secret_backend_key.vso_client_cache.name}"
-    capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+    capabilities = ["create", "update"]
-    description  = "admin privileges"
+  }
+  rule {
+    path = "${vault_mount.transit.path}/decrypt/${vault_transit_secret_backend_key.vso_client_cache.name}"
+    capabilities = ["create", "update"]
   }
 }
-resource "vault_policy" "admin" {
+resource "vault_policy" "vso_client_cache" {
-    name = "admin"
+    name = "edit-vso-client-cache"
-    policy = data.vault_policy_document.admin.hcl
+    policy = data.vault_policy_document.vso_client_cache.hcl
+}
+
+resource "vault_kubernetes_auth_backend_role" "vso" {
+  backend                          = vault_auth_backend.kubernetes.path
+  role_name                        = "vault-secret-operator"
+  bound_service_account_names      = ["hashicorp-vault-vault-secrets-operator-controller-manager"]
+  bound_service_account_namespaces = ["tools"]
+  token_ttl                        = 0
+  token_period                     = 120
+  token_policies                   = ["default", vault_policy.vso_client_cache.name]
+  audience                         = "vault"
+  alias_name_source                = "serviceaccount_name"
+}
+
+locals { # turn into *.tfvars ?
+  apps = toset([
+    "webapp",
+  ])
+}
+module "app_policies" {
+  source = "./modules/app_policy"
+  for_each = local.apps
+  name = each.value
+  gitea_app_id = var.gitea_app_id
 }

hashicorp-vault/iac/modules/app_policy/main.tf

@@ -0,0 +1,138 @@
+
+# jwt cicd app ops
+# allow creating
+# - vault role
+# - backend k8s
+# - postgres role
+
+locals {
+  name = lower(var.name)
+}
+
+data "vault_policy_document" "ops" {
+
+  # use terraform vault provider
+  rule {
+    path = "auth/token/create"
+    capabilities = ["create","update"]
+  }
+
+  # check on mounted auth backend (such as k8s)
+  rule {
+    path = "sys/mounts/auth/*"
+    capabilities = [ "read" ]
+  }
+  # read google credentials for terraform gcs backend
+  rule {
+    path = "kvv1/google/credentials"
+    capabilities = [ "read" ]
+  }
+
+  # edit postgres credentials access permissions
+  rule {
+    path = "postgres/roles/${local.name}*"
+    capabilities = [ "read", "list", "create", "update", "delete" ]
+  }
+  # edit k8s role
+  rule {
+    path = "auth/kubernetes/role/${local.name}*"
+    capabilities = [ "read", "list", "create", "update", "delete" ]
+    allowed_parameter {
+      key = "*"
+      value = []
+    }
+    allowed_parameter {
+      key = "bound_service_account_names"
+      value = [ jsonencode([local.name]) ]
+    }
+    allowed_parameter {
+      key = "bound_service_account_namespaces"
+      value = [ jsonencode([local.name]) ]
+    }
+    allowed_parameter {
+      key = "token_policies"
+      value = [
+        jsonencode(["default", local.name]),
+        jsonencode([local.name, "default"])
+      ]
+    }
+    
+  }
+  # allow editing app secrets
+  rule {
+    path = "kvv2/data/${local.name}/*"
+    capabilities = [ "create", "update", "read", "delete" ]
+  }
+  rule {
+    path = "kvv2/delete/${local.name}/*"
+    capabilities = [ "update" ]
+  }
+  rule {
+    path = "kvv2/undelete/${local.name}/*"
+    capabilities = [ "update" ]
+  }
+  rule {
+    path = "kvv2/destroy/${local.name}/*"
+    capabilities = [ "update" ]
+  }
+  rule {
+    path = "kvv2/metadata/${local.name}/*"
+    capabilities = [ "read", "list", "delete" ]
+  }
+  # allow edit vault role (risky ?)
+}
+resource "vault_policy" "ops" {
+  name = "${local.name}-ops"
+  /*
+    allowed_parameters = {
+  -   "bound_service_account_names" = ["["webapp"]"]
+  +   "bound_service_account_names" = [["webapp"]]
+    }
+  */
+  
+  policy = replace(
+    replace(
+      data.vault_policy_document.ops.hcl,
+      "\"[\"", "[\""
+    ),
+    "\"]\"", "\"]"
+  )
+}
+
+resource "vault_identity_group" "ops" {
+  name     = "${local.name}-ops"
+  type     = "internal"
+  external_member_entity_ids = true
+  policies = [vault_policy.ops.name]
+}
+
+data "vault_auth_backend" "gitea_jwt" {
+  path = "gitea_jwt"
+}
+resource "vault_jwt_auth_backend_role" "gitea_jwt_cicd" {
+  backend         = data.vault_auth_backend.gitea_jwt.path
+  role_name       = "gitea_cicd_${local.name}"
+  token_policies  = ["default"] # give "${local.name}-ops" role to group of entities
+
+  bound_audiences = [
+    var.gitea_app_id,
+  ]
+  
+  user_claim      = "email"
+  role_type       = "jwt"
+}
+
+data "vault_policy_document" "app" {
+  rule {
+    path         = "kvv2/data/${local.name}/*"
+    capabilities = ["read", "list"]
+  }
+  rule {
+    path         = "postgres/creds/${local.name}*"
+    capabilities = ["read"]
+  }
+}
+resource "vault_policy" "app" {
+    name = "${local.name}"
+    policy = data.vault_policy_document.app.hcl
+}

hashicorp-vault/iac/modules/app_policy/variables.tf

@@ -0,0 +1,6 @@
+variable "name" {
+  type = string
+}
+variable "gitea_app_id" {
+  type = string
+}

hashicorp-vault/iac/variables.tf

@@ -0,0 +1,11 @@
+variable "gitea_app_id" {
+  type = string
+}
+variable "POSTGRES_CREDENTIALS_EDITOR_USERNAME" {
+  type = string
+  sensitive = true
+}
+variable "POSTGRES_CREDENTIALS_EDITOR_PASSWORD" {
+  type = string
+  sensitive = true
+}

hashicorp-vault/values.yaml

@@ -6,6 +6,8 @@ vault: &vault_config
   server:
     enabled: true
     logLevel: trace
+    auditStorage:
+      enabled: true
   
     ingress:
       enabled: true
@@ -50,6 +52,26 @@ vault: &vault_config
     enabled: true
     annotations: {}
 
+vault-secrets-operator:
+  defaultVaultConnection:
+    enabled: true
+    address: http://hashicorp-vault.tools.svc.cluster.local:8200
+  defaultAuthMethod:
+    enabled: true
+  controller:
+    manager:
+      clientCache:
+        persistenceModel: direct-encrypted
+        storageEncryption:
+          enabled: true
+          mount: vault-secret-operator
+          keyName: vso-client-cache
+          transitMount: transit
+          kubernetes:
+            role: edit-vso-client-cache
+            serviceAccount: hashicorp-vault-vault-secrets-operator-controller-manager
+
+
 
 tool:
   # kind: 'SubChart' or 'HelmChart', if subchart then uncomment Chart.yaml dependency, else comment and use tool library with helm chart template