From 2a294eb2730cc04b6bee2ccc0355e3d288ba38ed Mon Sep 17 00:00:00 2001 From: Taqi Tahmid Date: Thu, 26 Jun 2025 21:01:12 +0300 Subject: [PATCH] kubernetes: added additional ingress controller for internal access - added additional traefik ingress controller for accessing internal services via ingress. --- kubernetes/README.md | 26 ++++++- .../Chart.yaml | 0 .../templates/clusterIssuers.yaml | 0 .../templates/secret-cloudflare.yaml | 0 .../templates/wildcardCert.yaml | 0 .../values.yaml | 0 .../docker-registry-helm-chart/values.yaml | 2 +- kubernetes/immich/values.yaml | 6 +- kubernetes/minio/values-tenant.yaml | 68 +++++++++---------- .../my-portfolio/portfolioManifest.yaml | 2 +- .../traefik/traefik-internal/values.yaml | 6 ++ .../traefik-middleware/ip-whitelist.yaml | 7 +- 12 files changed, 72 insertions(+), 45 deletions(-) rename kubernetes/{cert-manager-helm-chart => cert-manager-config-helm-chart}/Chart.yaml (100%) rename kubernetes/{cert-manager-helm-chart => cert-manager-config-helm-chart}/templates/clusterIssuers.yaml (100%) rename kubernetes/{cert-manager-helm-chart => cert-manager-config-helm-chart}/templates/secret-cloudflare.yaml (100%) rename kubernetes/{cert-manager-helm-chart => cert-manager-config-helm-chart}/templates/wildcardCert.yaml (100%) rename kubernetes/{cert-manager-helm-chart => cert-manager-config-helm-chart}/values.yaml (100%) create mode 100644 kubernetes/traefik/traefik-internal/values.yaml diff --git a/kubernetes/README.md b/kubernetes/README.md index 04e3f4b..bf74329 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -1,6 +1,6 @@ # Setup K3s Kubernetes Cluster -# Configure Traefik with extra values +# Configure Traefik Ingress Controller The Traefik ingress controller is deployed along with K3s. To modify the default values, @@ -12,6 +12,26 @@ helm upgrade traefik traefik/traefik \ --version 22.1.0 ``` +## Additional Ingress Controller for Internal Access + +An additional ingress controller is deployed for internal access to services. +This ingress controller is used to access services that are not exposed to the +internet. It is deployed in the `internal-ingress` namespace and uses the +Traefik ingress controller. + +To utilize the internal ingress controller, add the following +`ingressClassName: traefik-internal` under ingress spec. + +```bash +helm upgrade --install \ + --create-namespace traefik-internal traefik/traefik \ + --namespace traefik-internal \ + -f traefik/traefik-internal/values.yaml +``` + +The LoadBalancer service IP for the internal ingress controller is added to +the adGuard DNS server to resolve the internal services. + # Configure Cert Manager for automating SSL certificate handling Cert manager handles SSL certificate creation and renewal from Let's Encrypt. @@ -50,11 +70,11 @@ export KUBE_EDITOR=nvim kubectl -n kube-system edit configmap coredns ``` -Next, deploy the ClusterIssuer, WildcardCert, and secrets using helm +Next, deploy the ClusterIssuer, WildcardCert, and secrets using helm chart. ```bash source .env -helm install cert-handler cert-manager-helm-chart \ +helm install cert-handler cert-manager-config-helm-chart \ --atomic --set secret.apiToken=$CLOUDFLARE_TOKEN \ --set clusterIssuer.email=$EMAIL \ --set wildcardCert.dnsNames[0]=$DNSNAME diff --git a/kubernetes/cert-manager-helm-chart/Chart.yaml b/kubernetes/cert-manager-config-helm-chart/Chart.yaml similarity index 100% rename from kubernetes/cert-manager-helm-chart/Chart.yaml rename to kubernetes/cert-manager-config-helm-chart/Chart.yaml diff --git a/kubernetes/cert-manager-helm-chart/templates/clusterIssuers.yaml b/kubernetes/cert-manager-config-helm-chart/templates/clusterIssuers.yaml similarity index 100% rename from kubernetes/cert-manager-helm-chart/templates/clusterIssuers.yaml rename to kubernetes/cert-manager-config-helm-chart/templates/clusterIssuers.yaml diff --git a/kubernetes/cert-manager-helm-chart/templates/secret-cloudflare.yaml b/kubernetes/cert-manager-config-helm-chart/templates/secret-cloudflare.yaml similarity index 100% rename from kubernetes/cert-manager-helm-chart/templates/secret-cloudflare.yaml rename to kubernetes/cert-manager-config-helm-chart/templates/secret-cloudflare.yaml diff --git a/kubernetes/cert-manager-helm-chart/templates/wildcardCert.yaml b/kubernetes/cert-manager-config-helm-chart/templates/wildcardCert.yaml similarity index 100% rename from kubernetes/cert-manager-helm-chart/templates/wildcardCert.yaml rename to kubernetes/cert-manager-config-helm-chart/templates/wildcardCert.yaml diff --git a/kubernetes/cert-manager-helm-chart/values.yaml b/kubernetes/cert-manager-config-helm-chart/values.yaml similarity index 100% rename from kubernetes/cert-manager-helm-chart/values.yaml rename to kubernetes/cert-manager-config-helm-chart/values.yaml diff --git a/kubernetes/docker-registry-helm-chart/values.yaml b/kubernetes/docker-registry-helm-chart/values.yaml index fae5141..ce0fe6d 100644 --- a/kubernetes/docker-registry-helm-chart/values.yaml +++ b/kubernetes/docker-registry-helm-chart/values.yaml @@ -14,7 +14,7 @@ ingress: enabled: true annotations: traefik.ingress.kubernetes.io/router.entrypoints: websecure - cert-manager.io/issuer: "letsencrypt-prod" + cert-manager.io/cluster-issuer: "acme-issuer" tls: enabled: true host: "*.example.com" diff --git a/kubernetes/immich/values.yaml b/kubernetes/immich/values.yaml index 75e43df..972fdf4 100644 --- a/kubernetes/immich/values.yaml +++ b/kubernetes/immich/values.yaml @@ -63,7 +63,7 @@ server: enabled: true annotations: traefik.ingress.kubernetes.io/router.entrypoints: websecure - cert-manager.io/issuer: "letsencrypt-prod" + cert-manager.io/cluster-issuer: "acme-issuer" hosts: - host: placeholder.immich.app paths: @@ -74,7 +74,7 @@ server: - placeholder.immich.app machine-learning: - enabled: false # disabled due to resource constraints + enabled: false # disabled due to resource constraints image: repository: ghcr.io/immich-app/immich-machine-learning pullPolicy: IfNotPresent @@ -86,4 +86,4 @@ machine-learning: size: 10Gi # Optional: Set this to pvc to avoid downloading the ML models every start. type: pvc - accessMode: ReadWriteMany \ No newline at end of file + accessMode: ReadWriteMany diff --git a/kubernetes/minio/values-tenant.yaml b/kubernetes/minio/values-tenant.yaml index e817795..704a9f0 100644 --- a/kubernetes/minio/values-tenant.yaml +++ b/kubernetes/minio/values-tenant.yaml @@ -36,7 +36,7 @@ tenant: # # An array of Kubernetes secrets to use for pulling images from a private ``image.repository``. # Only one array element is supported at this time. - imagePullSecret: { } + imagePullSecret: {} ### # # Specify `initContainers `__ to perform setup or configuration tasks before the main Tenant pods starts. @@ -60,12 +60,12 @@ tenant: # done # echo "IdP (${URL}) reachable. Starting MinIO..." # - initContainers: [ ] + initContainers: [] ### # The Kubernetes `Scheduler `__ to use for dispatching Tenant pods. # # Specify an empty dictionary ``{}`` to dispatch pods with the default scheduler. - scheduler: { } + scheduler: {} ### # Root key for dynamically creating a secret for use with configuring root MinIO User # Specify the ``name`` and then a list of environment variables. @@ -94,10 +94,10 @@ tenant: poolsMetadata: ### # Specify `annotations `__ to associate to Tenant pods. - annotations: { } + annotations: {} ### # Specify `labels `__ to associate to Tenant pods. - labels: { } + labels: {} ### # If this variable is set to true, then enable the usage of an existing Kubernetes secret to set environment variables for the Tenant. @@ -142,42 +142,42 @@ tenant: storageClassName: longhorn-2x ### # Specify `storageAnnotations `__ to associate to PVCs. - storageAnnotations: { } + storageAnnotations: {} ### # Specify `storageLabels `__ to associate to PVCs. - storageLabels: { } + storageLabels: {} ### # Specify `annotations `__ to associate to Tenant pods. - annotations: { } + annotations: {} ### # Specify `labels `__ to associate to Tenant pods. - labels: { } + labels: {} ### # # An array of `Toleration labels `__ to associate to Tenant pods. # # These settings determine the distribution of pods across worker nodes. - tolerations: [ ] + tolerations: [] ### # Any `Node Selectors `__ to apply to Tenant pods. # # The Kubernetes scheduler uses these selectors to determine which worker nodes onto which it can deploy Tenant pods. # # If no worker nodes match the specified selectors, the Tenant deployment will fail. - nodeSelector: { } + nodeSelector: {} ### # # The `affinity `__ or anti-affinity settings to apply to Tenant pods. # # These settings determine the distribution of pods across worker nodes and can help prevent or allow colocating pods onto the same worker nodes. - affinity: { } + affinity: {} ### # # The `Requests or Limits `__ for resources to associate to Tenant pods. # # These settings can control the minimum and maximum resources requested for each pod. # If no worker nodes can meet the specified requests, the Operator may fail to deploy. - resources: { } + resources: {} ### # The Kubernetes `SecurityContext `__ to use for deploying Tenant resources. # @@ -208,7 +208,7 @@ tenant: # An array of `Topology Spread Constraints `__ to associate to Operator Console pods. # # These settings determine the distribution of pods across worker nodes. - topologySpreadConstraints: [ ] + topologySpreadConstraints: [] ### # # The name of a custom `Container Runtime `__ to use for the Operator Console pods. @@ -240,7 +240,7 @@ tenant: # This is used by MinIO to verify TLS connections from clients using those CAs # If you omit this and have clients using TLS certificates minted by an external CA, those connections may fail with warnings around certificate verification. # See `Operator CRD: TenantSpec `__. - externalCaCertSecret: [ ] + externalCaCertSecret: [] ### # Specify an array of Kubernetes secrets, where each entry corresponds to a secret contains the TLS private key and public certificate pair. # @@ -257,7 +257,7 @@ tenant: # You can pass the CA to the Operator to allow it to trust that cert. # See `Self-Signed, Internal, and Private Certificates `__ for more information. # This step may also be necessary for globally trusted CAs where you must provide intermediate certificates to the Operator to help build the full chain of trust. - externalCertSecret: [ ] + externalCertSecret: [] ### # Enable automatic Kubernetes based `certificate generation and signing `__ requestAutoCert: false @@ -272,13 +272,13 @@ tenant: # The default DNS name format is typically ``*.minio.default.svc.cluster.local``. # # See `Operator CRD: CertificateConfig `__ - certConfig: { } + certConfig: {} ### # MinIO features to enable or disable in the MinIO Tenant # See `Operator CRD: Features `__. features: bucketDNS: false - domains: { } + domains: {} enableSFTP: false ### # Array of objects describing one or more buckets to create during tenant provisioning. @@ -289,31 +289,31 @@ tenant: # - name: my-minio-bucket # objectLock: false # optional # region: us-east-1 # optional - buckets: [ ] + buckets: [] ### # Array of Kubernetes secrets from which the Operator generates MinIO users during tenant provisioning. # # Each secret should specify the ``CONSOLE_ACCESS_KEY`` and ``CONSOLE_SECRET_KEY`` as the access key and secret key for that user. - users: [ ] + users: [] ### # The `PodManagement `__ policy for MinIO Tenant Pods. # Can be "OrderedReady" or "Parallel" podManagementPolicy: Parallel # The `Liveness Probe `__ for monitoring Tenant pod liveness. # Tenant pods will be restarted if the probe fails. - liveness: { } + liveness: {} ### # `Readiness Probe `__ for monitoring Tenant container readiness. # Tenant pods will be removed from service endpoints if the probe fails. - readiness: { } + readiness: {} ### # `Startup Probe `__ for monitoring container startup. # Tenant pods will be restarted if the probe fails. # Refer - startup: { } + startup: {} ### # The `Lifecycle hooks `__ for container. - lifecycle: { } + lifecycle: {} ### # Directs the Operator to deploy the MinIO S3 API and Console services as LoadBalancer objects. # @@ -345,14 +345,14 @@ tenant: # # $ k logs myminio-pool-0-0 -n default # {"level":"INFO","errKind":"","time":"2022-04-07T21:49:33.740058549Z","message":"All MinIO sub-systems initialized successfully"} - logging: { } + logging: {} ### # serviceMetadata allows passing additional labels and annotations to MinIO and Console specific # services created by the operator. - serviceMetadata: { } + serviceMetadata: {} ### # Add environment variables to be set in MinIO container (https://github.com/minio/minio/tree/master/docs/config) - env: [ ] + env: [] ### # PriorityClassName indicates the Pod priority and hence importance of a Pod relative to other Pods. # This is applied to MinIO pods only. @@ -362,7 +362,7 @@ tenant: # An array of `Volumes `__ which the Operator can mount to Tenant pods. # # The volumes must exist *and* be accessible to the Tenant pods. - additionalVolumes: [ ] + additionalVolumes: [] ### # An array of volume mount points associated to each Tenant container. # @@ -375,7 +375,7 @@ tenant: # mountPath: /path/to/mount # # The ``name`` field must correspond to an entry in the ``additionalVolumes`` array. - additionalVolumeMounts: [ ] + additionalVolumeMounts: [] # Define configuration for KES (stateless and distributed key-management system) # Refer https://github.com/minio/kes #kes: @@ -483,7 +483,7 @@ ingress: api: enabled: false ingressClassName: - labels: { } + labels: {} annotations: tls: - secretName: minio-tls @@ -494,11 +494,11 @@ ingress: pathType: Prefix console: enabled: true - ingressClassName: "traefik" - labels: { } + ingressClassName: "traefik-internal" + labels: {} annotations: traefik.ingress.kubernetes.io/router.entrypoints: websecure - cert-manager.io/issuer: "letsencrypt-prod" + cert-manager.io/cluster-issuer: "acme-issuer" traefik.ingress.kubernetes.io/service.serversTransport: insecure-transport traefik.ingress.kubernetes.io/router.middlewares: kube-system-ip-whitelist@kubernetescrd tls: @@ -520,4 +520,4 @@ ingress: # stringData: # config.env: |- # export MINIO_ROOT_USER='minio' -# export MINIO_ROOT_PASSWORD='minio123' \ No newline at end of file +# export MINIO_ROOT_PASSWORD='minio123' diff --git a/kubernetes/my-portfolio/portfolioManifest.yaml b/kubernetes/my-portfolio/portfolioManifest.yaml index 2e82397..551d13c 100644 --- a/kubernetes/my-portfolio/portfolioManifest.yaml +++ b/kubernetes/my-portfolio/portfolioManifest.yaml @@ -44,7 +44,7 @@ metadata: name: portfolio annotations: traefik.ingress.kubernetes.io/router.entrypoints: websecure - cert-manager.io/issuer: "letsencrypt-prod" + cert-manager.io/cluster-issuer: "acme-issuer" spec: tls: - hosts: diff --git a/kubernetes/traefik/traefik-internal/values.yaml b/kubernetes/traefik/traefik-internal/values.yaml new file mode 100644 index 0000000..896ad72 --- /dev/null +++ b/kubernetes/traefik/traefik-internal/values.yaml @@ -0,0 +1,6 @@ +ingressClass: + enabled: true + isDefaultClass: false + name: traefik-internal +additionalArguments: + - "--providers.kubernetesingress.ingressclass=traefik-internal" \ No newline at end of file diff --git a/kubernetes/traefik/traefik-middleware/ip-whitelist.yaml b/kubernetes/traefik/traefik-middleware/ip-whitelist.yaml index 25fa616..77390f8 100644 --- a/kubernetes/traefik/traefik-middleware/ip-whitelist.yaml +++ b/kubernetes/traefik/traefik-middleware/ip-whitelist.yaml @@ -2,9 +2,10 @@ apiVersion: traefik.io/v1alpha1 kind: Middleware metadata: name: ip-whitelist - namespace: minio + namespace: kube-system spec: ipWhiteList: sourceRange: - - 192.168.1.0/24 - - 87.92.7.212/32 + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16