workflow dispatch trigger

.gitea/workflows/helmcharts.yaml

@@ -2,7 +2,7 @@
 # template source: https://github.com/bretfisher/docker-build-workflow/blob/main/templates/call-docker-build.yaml
 name: Helm Charts
 
-on: [push,pull_request]
+on: [push,pull_request,workflow_dispatch]
   # push: &helmPaths # turns out gitea don't handle well the paths filter
   #   paths:
   #     - '*/\.yaml'

.gitea/workflows/vault.yaml

@@ -0,0 +1,63 @@
+---
+# template source: https://github.com/bretfisher/docker-build-workflow/blob/main/templates/call-docker-build.yaml
+name: Hashicorp Vault
+
+on: #[push,pull_request]
+  workflow_dispatch: {}
+  push: &vaultPaths
+    paths:
+      - 'hashicorp-vault/**/*.tf'
+  pull_request: *vaultPaths
+
+# cancel any previously-started, yet still active runs of this workflow on the same branch
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+.vault_step: &vault_step
+  name: read vault secret
+  uses: https://gitea.arcodange.duckdns.org/arcodange-org/vault-action.git@main
+  id: vault-secrets
+  with:
+    url: https://vault.arcodange.duckdns.org
+    jwtGiteaOIDC: ${{ needs.gitea_vault_auth.outputs.gitea_vault_jwt }}
+    role: gitea_cicd
+    method: jwt
+    path: gitea_jwt
+    secrets: |
+        kvv1/google/credentials credentials | GOOGLE_BACKEND_CREDENTIALS ;
+        kvv1/gitea/app id | TF_VAR_gitea_app_id ;
+        kvv1/postgres/credentials_editor/credentials * | TF_VAR_POSTGRES_CREDENTIALS_EDITOR_ ;
+
+jobs:
+  gitea_vault_auth:
+    name: Auth with gitea for vault
+    runs-on: ubuntu-latest
+    outputs:
+      gitea_vault_jwt: ${{steps.gitea_vault_jwt.outputs.id_token}}
+    steps:
+
+      - name: Auth with gitea for vault
+        id: gitea_vault_jwt
+        run: |
+          echo -n "${{ secrets.vault_oauth__sh_b64 }}" | base64 -d | bash
+
+  tofu:
+    name: Tofu - Vault IAC
+    needs:
+      - gitea_vault_auth
+    runs-on: ubuntu-latest
+    env:
+      OPENTOFU_VERSION: 1.8.2
+      TERRAFORM_VAULT_AUTH_JWT: ${{ needs.gitea_vault_auth.outputs.gitea_vault_jwt }}
+    steps:
+      - *vault_step
+      - uses: actions/checkout@v4
+      # - uses: dflook/terraform-plan@v1
+      #   with:
+      #     path: hashicorp-vault/iac
+      - name: terraform apply
+        uses: dflook/terraform-apply@v1
+        with:
+          path: hashicorp-vault/iac
+          auto_approve: true

.gitignore

@@ -1,3 +1,5 @@
 .DS_Store
 Chart.lock
 */charts/*.tgz
+.terraform
+.terraform.lock.hcl

chart/templates/project.yaml

@@ -22,4 +22,6 @@ spec:
     - group: '*'
       kind: ClusterRole
     - group: '*'
-      kind: MutatingWebhookConfiguration
+      kind: MutatingWebhookConfiguration
+    - group: 'apiextensions.k8s.io'
+      kind: CustomResourceDefinition

hashicorp-vault/.helmignore

@@ -21,3 +21,7 @@
 .idea/
 *.tmproj
 .vscode/
+iac/
+README.md
+.terraform
+.terraform.lock

hashicorp-vault/Chart.yaml

@@ -9,6 +9,9 @@ dependencies:
 - name: vault
   version: 0.28.1
   repository: https://helm.releases.hashicorp.com
+- name: vault-secrets-operator
+  version: 0.9.0
+  repository: https://helm.releases.hashicorp.com
 
 # A chart can be either an 'application' or a 'library' chart.
 #

hashicorp-vault/README.md

@@ -0,0 +1,151 @@
+# Vault
+
+1. Les [playbooks ansible](https://gitea.arcodange.duckdns.org/arcodange-org/factory/src/branch/main/ansible/arcodange/factory/playbooks) configurent la base de données postgres et le minimum requis pour permetre au dépot "tools" d'appliquer via un workflow gitea action [une configuration vault via tofu](./iac/).
+2. Configuration des backend d'authentification et des roles pour postgres et kubernetes. Définition de rôles "${app}-ops" pour permettre au dépot d'une application de définir ses propres dépendances dans vault. Rotation de credentials postgres pour les applications.
+3. [Le dépot de l'application webapp](https://gitea.arcodange.duckdns.org/arcodange-org/webapp) gère l'obtention de ses crédentials pour postgres.
+
+```mermaid
+flowchart LR
+
+    classDef playbook stroke:#0f0,fill:#440,stroke-width: 1px
+    classDef tofu stroke:#f00,fill:#404,stroke-width: 2px
+    classDef argocd stroke:#00f,fill:#044,stroke-width: 3px
+    classDef database stroke:#bb0,fill:#ff0,stroke-width: 5px,color: black
+    classDef secret stroke:#f00,fill:#f00,stroke-width: 5px,color: yellow
+    classDef secretOperator stroke:#f00,fill:DarkRed,stroke-width: 5px,color: Orange
+
+    subgraph git_code[factory.git]
+        subgraph ansible_collection
+            setup_playbook[playbook arcodange.factory.setup]:::playbook
+            tools_playbook[playbook arcodange.factory.tools]:::playbook
+        end
+        git_code_tofu_vault{{tofu}}:::tofu
+    end
+
+    subgraph git_tools[tools.git]
+        argocd_tools{{Argo CD Apps}}:::argocd
+        git_tools_tofu_vault{{tofu}}:::tofu
+    end
+
+    subgraph git_webapp[webapp.git]
+        webapp["Go(lang) web app"]
+        argocd_webapp{{Argo CD App}}:::argocd
+        git_webapp_tofu_vault{{tofu}}:::tofu
+    end
+
+    subgraph servers
+        subgraph k3s
+            subgraph k3s_ns_tools[ns:tools]
+                argocd{{Argo CD}}:::argocd
+                pgbouncer
+                subgraph vault
+                    subgraph vault_auth[auth]
+                        subgraph vault_auth_openid[openid]
+                        end
+                        vault_auth_jwt[jwt]
+                        vault_auth_k8s[kubernetes]
+                        vault_auth_jwt_role_gitea_cicd[gitea_cicd role]
+                        vault_auth_jwt_role_gitea_cicd_webapp_ops[gitea_cicd_webapp ops role]
+                        vault_auth_k8s_role_vso[vault-secret-operator role]
+                        vault_auth_k8s_role_webapp[webapp role]
+                        subgraph policies
+                            policy_default[default]
+                            policy_webapp[webapp]
+                            policy_webapp_ops[webapp ops]
+                            policy_admin[admin]
+                            policy_vso[edit-vso-client-cache]
+                        end
+                    end
+                    subgraph vault_secrets[secrets]
+                        subgraph kvv2
+                            google/credentials
+                            webapp/config
+                        end
+                    end
+                    subgraph vault_postgres[postgres]
+                        creds/creds-editor
+                        creds/webapp
+                    end
+                    subgraph vault_transit[transit]
+                    end
+                end
+                vault-secret-operator:::secretOperator
+            end
+            subgraph k3s_ns_webapp[ns:webapp]
+                webapp_deployment[deployment:webapp]
+                webapp_postgres_creds_secret[secret:postgres creds]:::secret
+                webapp_config_secret[secret:config]:::secret
+                webapp_service_account[sa:webapp]
+            end
+        end
+        subgraph postgres
+            root_credentials
+            postgres_db[(postgres)]:::database
+            webapp_credentials:::secret
+            webapp_db[(webpp)]:::database
+
+            vault_creds_editor_role{{credentials_editor}}
+        end
+    end
+
+    setup_playbook -. setup postgres .-> postgres
+    
+    tools_playbook -.-o git_code_tofu_vault
+    git_code_tofu_vault -..-> vault_auth_openid
+    git_code_tofu_vault -..-> vault_auth_jwt -- tofu:factory --- vault_auth_jwt_role_gitea_cicd
+    git_code_tofu_vault -..-> kvv2
+    git_code_tofu_vault -..-> google/credentials
+
+    linkStyle 0,1 stroke:#ff3,stroke-width:1px,color:DarkKhaki;
+    linkStyle 2,3,5,6 stroke:#f3f,stroke-width:2px,color:DarkOrange;
+    
+    git_tools -.-o argocd_tools
+    argocd_tools -.-> pgbouncer
+    argocd_tools -.-> vault
+    argocd_tools -.-> vault-secret-operator
+    argocd_tools o--o argocd
+
+    linkStyle 7,8,9,10,11 stroke:#3ff,stroke-width:3px,color:DarkSlateBlue;
+
+    git_tools_tofu_vault -..-> vault_auth_k8s -- sa:vso --- vault_auth_k8s_role_vso
+    git_tools_tofu_vault -..-> webapp/config
+    git_tools_tofu_vault -..-> vault_transit
+    git_tools_tofu_vault -..-> vault_postgres
+    vault_auth_k8s ---> k3s
+    vault_postgres --> pgbouncer x==> postgres; webapp_deployment --> pgbouncer
+
+    linkStyle 12,14,15,16 stroke:#f3f,stroke-width:2px,color:DarkOrange;
+    linkStyle 18,19,20 stroke:gold,stroke-width:2px;
+    
+    vault_transit x---x vault-secret-operator
+    vault-secret-operator x---x vault_auth_k8s_role_vso
+
+    vault_auth_jwt_role_gitea_cicd x--x policy_default
+    vault_auth_k8s_role_vso x--x policy_vso
+
+    creds/webapp -.-> webapp_credentials
+    creds/webapp -.-> vault-secret-operator
+    vault-secret-operator -.-> webapp_postgres_creds_secret
+    webapp/config -.-> vault-secret-operator
+    vault-secret-operator -.-> webapp_config_secret
+
+    argocd_webapp -.-> k3s_ns_webapp
+    webapp --o webapp_deployment
+    webapp_postgres_creds_secret --o webapp_deployment
+    webapp_deployment --> webapp_service_account
+
+    vault_auth_jwt -- tofu:tools --- vault_auth_jwt_role_gitea_cicd_webapp_ops
+    vault_auth_jwt_role_gitea_cicd_webapp_ops x--x policy_webapp_ops
+    vault_auth_k8s -- sa:webapp --- vault_auth_k8s_role_webapp x-- tofu:webapp --x policy_webapp
+
+    git_webapp_tofu_vault -.-> vault_auth_k8s_role_webapp
+    git_webapp_tofu_vault -.-> creds/webapp
+
+
+    root_credentials x--x postgres_db
+    webapp_credentials x--x webapp_db
+
+    tools_playbook --> vault_creds_editor_role
+    vault_creds_editor_role -. change password .-> webapp_credentials
+    vault_postgres x--x vault_creds_editor_role
+```

hashicorp-vault/iac/backend.tf

@@ -0,0 +1,6 @@
+terraform {
+  backend "gcs" {
+    bucket  = "arcodange-tf"
+    prefix  = "tools/hashicorp_vault/main"
+  }
+}

hashicorp-vault/iac/main.tf

@@ -0,0 +1,82 @@
+resource "vault_auth_backend" "kubernetes" {
+  type = "kubernetes"
+}
+resource "vault_kubernetes_auth_backend_config" "config" {
+  backend                = vault_auth_backend.kubernetes.path
+  kubernetes_host        = "https://kubernetes.default.svc:443"
+}
+
+resource "vault_mount" "kvv2" {
+  path        = "kvv2"
+  type        = "kv"
+  options     = { version = "2" }
+  description = "KV Version 2 secret engine mount"
+}
+
+
+resource "vault_mount" "postgres" {
+  path = "postgres"
+  type = "database"
+}
+
+resource "vault_database_secret_backend_connection" "postgres" {
+  backend       = vault_mount.postgres.path
+  name          = "postgres"
+  allowed_roles = ["*"]
+  root_rotation_statements = [
+    "ALTER USER \"{{name}}\" WITH PASSWORD '{{password}}';",
+  ]
+
+  postgresql {
+    connection_url = "postgresql://{{username}}:{{password}}@pgbouncer.tools:5432/postgres?sslmode=disable"
+    username = var.POSTGRES_CREDENTIALS_EDITOR_USERNAME
+    password = var.POSTGRES_CREDENTIALS_EDITOR_PASSWORD
+  }
+}
+
+
+resource "vault_mount" "transit" {
+  path                      = "transit"
+  type                      = "transit"
+  description               = "Pour le vault secret operator (vso) dans k3s en cas de redemarrage par exemple"
+#   default_lease_ttl_seconds = 3600
+#   max_lease_ttl_seconds     = 86400
+}
+resource "vault_transit_secret_backend_key" "vso_client_cache" {
+  backend = vault_mount.transit.path
+  name    = "vso-client-cache"
+}
+
+data "vault_policy_document" "vso_client_cache" {
+  rule {
+    path = "${vault_mount.transit.path}/encrypt/${vault_transit_secret_backend_key.vso_client_cache.name}"
+    capabilities = ["create", "update"]
+  }
+  rule {
+    path = "${vault_mount.transit.path}/decrypt/${vault_transit_secret_backend_key.vso_client_cache.name}"
+    capabilities = ["create", "update"]
+  }
+}
+resource "vault_policy" "vso_client_cache" {
+    name = "edit-vso-client-cache"
+    policy = data.vault_policy_document.vso_client_cache.hcl
+}
+
+resource "vault_kubernetes_auth_backend_role" "vso" {
+  backend                          = vault_auth_backend.kubernetes.path
+  role_name                        = "vault-secret-operator"
+  bound_service_account_names      = ["hashicorp-vault-vault-secrets-operator-controller-manager"]
+  bound_service_account_namespaces = ["tools"]
+  token_ttl                        = 0
+  token_period                     = 120
+  token_policies                   = ["default", vault_policy.vso_client_cache.name]
+  audience                         = "vault"
+  alias_name_source                = "serviceaccount_name"
+}
+
+module "app_policies" {
+  source = "./modules/app_policy"
+  for_each = var.applications
+  name = each.value
+  gitea_app_id = var.gitea_app_id
+}

hashicorp-vault/iac/modules/app_policy/main.tf

@@ -0,0 +1,143 @@
+
+# jwt cicd app ops
+# allow creating
+# - vault role
+# - backend k8s
+# - postgres role
+
+locals {
+  name = lower(var.name)
+}
+
+data "vault_policy_document" "ops" {
+
+  # use terraform vault provider
+  rule {
+    path = "auth/token/create"
+    capabilities = ["create","update"]
+  }
+
+  # check on mounted auth backend (such as k8s)
+  rule {
+    path = "sys/mounts/auth/*"
+    capabilities = [ "read" ]
+  }
+  # read google credentials for terraform gcs backend
+  rule {
+    path = "kvv1/google/credentials"
+    capabilities = [ "read" ]
+  }
+  # read tofu_module_reader gitea bot user ssh keys
+  rule {
+    path = "kvv1/gitea/tofu_module_reader"
+    capabilities = [ "read" ]
+  }
+
+  # edit postgres credentials access permissions
+  rule {
+    path = "postgres/roles/${local.name}*"
+    capabilities = [ "read", "list", "create", "update", "delete" ]
+  }
+  # edit k8s role
+  rule {
+    path = "auth/kubernetes/role/${local.name}*"
+    capabilities = [ "read", "list", "create", "update", "delete" ]
+    allowed_parameter {
+      key = "*"
+      value = []
+    }
+    allowed_parameter {
+      key = "bound_service_account_names"
+      value = [ jsonencode([local.name]) ]
+    }
+    allowed_parameter {
+      key = "bound_service_account_namespaces"
+      value = [ jsonencode([local.name]) ]
+    }
+    allowed_parameter {
+      key = "token_policies"
+      value = [
+        jsonencode(["default", local.name]),
+        jsonencode([local.name, "default"])
+      ]
+    }
+    
+  }
+  # allow editing app secrets
+  rule {
+    path = "kvv2/data/${local.name}/*"
+    capabilities = [ "create", "update", "read", "delete" ]
+  }
+  rule {
+    path = "kvv2/delete/${local.name}/*"
+    capabilities = [ "update" ]
+  }
+  rule {
+    path = "kvv2/undelete/${local.name}/*"
+    capabilities = [ "update" ]
+  }
+  rule {
+    path = "kvv2/destroy/${local.name}/*"
+    capabilities = [ "update" ]
+  }
+  rule {
+    path = "kvv2/metadata/${local.name}/*"
+    capabilities = [ "read", "list", "delete" ]
+  }
+  # allow edit vault role (risky ?)
+}
+resource "vault_policy" "ops" {
+  name = "${local.name}-ops"
+  /*
+    allowed_parameters = {
+  -   "bound_service_account_names" = ["["webapp"]"]
+  +   "bound_service_account_names" = [["webapp"]]
+    }
+  */
+  
+  policy = replace(
+    replace(
+      data.vault_policy_document.ops.hcl,
+      "\"[\"", "[\""
+    ),
+    "\"]\"", "\"]"
+  )
+}
+
+resource "vault_identity_group" "ops" {
+  name     = "${local.name}-ops"
+  type     = "internal"
+  external_member_entity_ids = true
+  policies = [vault_policy.ops.name]
+}
+
+data "vault_auth_backend" "gitea_jwt" {
+  path = "gitea_jwt"
+}
+resource "vault_jwt_auth_backend_role" "gitea_jwt_cicd" {
+  backend         = data.vault_auth_backend.gitea_jwt.path
+  role_name       = "gitea_cicd_${local.name}"
+  token_policies  = ["default"] # give "${local.name}-ops" role to group of entities
+
+  bound_audiences = [
+    var.gitea_app_id,
+  ]
+  
+  user_claim      = "email"
+  role_type       = "jwt"
+}
+
+data "vault_policy_document" "app" {
+  rule {
+    path         = "kvv2/data/${local.name}/*"
+    capabilities = ["read", "list"]
+  }
+  rule {
+    path         = "postgres/creds/${local.name}*"
+    capabilities = ["read"]
+  }
+}
+resource "vault_policy" "app" {
+    name = "${local.name}"
+    policy = data.vault_policy_document.app.hcl
+}

hashicorp-vault/iac/modules/app_policy/variables.tf

@@ -0,0 +1,6 @@
+variable "name" {
+  type = string
+}
+variable "gitea_app_id" {
+  type = string
+}

hashicorp-vault/iac/modules/app_roles/main.tf

@@ -0,0 +1,38 @@
+data "vault_auth_backend" "kubernetes" {
+  path = "kubernetes"
+}
+
+locals {
+  name = lower(var.name)
+  database = var.database == null ? local.name : var.database
+
+  vault_mount_postgres = { path = "postgres" }
+  vault_mount_kvv2 = { path = "kvv2" }
+}
+
+resource "vault_database_secret_backend_role" "role" {
+  backend             = local.vault_mount_postgres.path
+  name                = "${local.name}"
+  db_name             = "postgres"
+  creation_statements = [
+    "CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';",
+    "GRANT ${local.name}_role TO \"{{name}}\";",
+  ] 
+  revocation_statements = [
+    "REASSIGN OWNED BY \"{{name}}\" TO ${local.name}_role;",
+    "REVOKE ALL ON DATABASE ${local.database} FROM  \"{{name}}\";", # should we drop the role ?
+  ]
+  renew_statements=[]
+  rollback_statements=[]
+}
+
+resource "vault_kubernetes_auth_backend_role" "role" {
+  backend                          = data.vault_auth_backend.kubernetes.path
+  role_name                        = local.name
+  bound_service_account_names      = [local.name]
+  bound_service_account_namespaces = [local.name]
+  token_ttl                        = 3600
+  token_policies                   = ["default", local.name]
+  audience                         = "vault"
+  alias_name_source                = "serviceaccount_name"
+}

hashicorp-vault/iac/modules/app_roles/outputs.tf

@@ -0,0 +1,16 @@
+output "name" {
+  value = local.name
+}
+output "database" {
+  value = local.database
+}
+output "mount_paths" {
+  value = {
+    k8s = data.vault_auth_backend.kubernetes.path
+    pg = local.vault_mount_postgres.path
+    kvv2 = local.vault_mount_kvv2.path
+  }
+}
+output "kvv2_path_prefix" {
+  value = format("%s/", local.name)
+}

hashicorp-vault/iac/modules/app_roles/providers.tf

@@ -0,0 +1,8 @@
+terraform {
+    required_providers {
+      vault = {
+        source = "vault"
+        version = ">= 4.4.0"
+      }
+    }
+}

hashicorp-vault/iac/modules/app_roles/variables.tf

@@ -0,0 +1,8 @@
+variable "name" {
+  type = string
+}
+variable "database" {
+  type = string
+  nullable = true
+  default = null
+}

hashicorp-vault/iac/providers.tf

@@ -0,0 +1,16 @@
+terraform {
+    required_providers {
+      vault = {
+        source = "vault"
+        version = "4.4.0"
+      }
+    }
+}
+
+provider vault {
+    address = "https://vault.arcodange.duckdns.org"
+    auth_login_jwt { # TERRAFORM_VAULT_AUTH_JWT environment variable
+        mount = "gitea_jwt"
+        role = "gitea_cicd"
+    }
+}

hashicorp-vault/iac/terraform.tfvars

@@ -0,0 +1,4 @@
+applications = [
+    "webapp",
+    "erp",
+]

hashicorp-vault/iac/variables.tf

@@ -0,0 +1,14 @@
+variable "gitea_app_id" {
+  type = string
+}
+variable "POSTGRES_CREDENTIALS_EDITOR_USERNAME" {
+  type = string
+  sensitive = true
+}
+variable "POSTGRES_CREDENTIALS_EDITOR_PASSWORD" {
+  type = string
+  sensitive = true
+}
+variable "applications" {
+  type = set(string)
+}

hashicorp-vault/values.yaml

@@ -5,7 +5,9 @@ vault: &vault_config
   
   server:
     enabled: true
-    logLevel: debug
+    logLevel: trace
+    auditStorage:
+      enabled: true
   
     ingress:
       enabled: true
@@ -50,6 +52,26 @@ vault: &vault_config
     enabled: true
     annotations: {}
 
+vault-secrets-operator:
+  defaultVaultConnection:
+    enabled: true
+    address: http://hashicorp-vault.tools.svc.cluster.local:8200
+  defaultAuthMethod:
+    enabled: true
+  controller:
+    manager:
+      clientCache:
+        persistenceModel: direct-encrypted
+        storageEncryption:
+          enabled: true
+          mount: vault-secret-operator
+          keyName: vso-client-cache
+          transitMount: transit
+          kubernetes:
+            role: edit-vso-client-cache
+            serviceAccount: hashicorp-vault-vault-secrets-operator-controller-manager
+
+
 
 tool:
   # kind: 'SubChart' or 'HelmChart', if subchart then uncomment Chart.yaml dependency, else comment and use tool library with helm chart template