From 5fd0b6cf8d9a645bd1893bf6d5cb8558e9eaac83 Mon Sep 17 00:00:00 2001 From: rguichard Date: Wed, 17 Jun 2026 10:51:03 +0200 Subject: [PATCH 01/12] chore(helm/charts): import erpc from morpho-infra-helm --- .github/CODEOWNERS | 7 + .github/workflows/helm-charts-validation.yaml | 150 ++ .github/workflows/slack-pr-notification.yaml | 49 + .github/workflows/wiz-iac-scan.yml | 35 + helm/charts/.gitkeep | 0 helm/charts/erpc/.env.example | 23 + helm/charts/erpc/Chart.yaml | 24 + helm/charts/erpc/README.md | 61 + helm/charts/erpc/docker-compose.yml | 14 + helm/charts/erpc/templates/_helpers.tpl | 87 + helm/charts/erpc/templates/deployment.yaml | 139 ++ helm/charts/erpc/templates/hpa.yaml | 38 + .../erpc/templates/image-catalog-pg18.yaml | 12 + .../erpc/templates/job-vault-config.yaml | 444 ++++ helm/charts/erpc/templates/job-vault-db.yaml | 89 + .../erpc/templates/monitoring-configmap.yaml | 59 + helm/charts/erpc/templates/pdb.yaml | 11 + helm/charts/erpc/templates/pg-erpc.yaml | 186 ++ .../erpc/templates/podmonitor-erpc-db.yaml | 17 + helm/charts/erpc/templates/podmonitor.yaml | 61 + helm/charts/erpc/templates/rbac-vault.yaml | 35 + .../erpc/templates/scheduled-backup.yaml | 20 + .../erpc/templates/service-headless.yaml | 23 + .../erpc/templates/service-metrics.yaml | 20 + helm/charts/erpc/templates/service.yaml | 19 + .../charts/erpc/templates/serviceaccount.yaml | 44 + helm/charts/erpc/test-local.sh | 273 +++ helm/charts/erpc/values.yaml | 260 +++ helm/environments/prd/.gitkeep | 0 helm/environments/prd/erpc/Chart.lock | 33 + helm/environments/prd/erpc/Chart.yaml | 50 + .../prd/erpc/charts/erpc-0.1.35.tgz | Bin 0 -> 16626 bytes .../prd/erpc/charts/haproxy-1.26.1.tgz | Bin 0 -> 32501 bytes .../prometheus-redis-exporter-6.8.0.tgz | Bin 0 -> 8636 bytes .../prd/erpc/charts/redis-ha-4.35.2.tgz | Bin 0 -> 36695 bytes .../prd/erpc/templates/_helpers.tpl | 23 + .../prd/erpc/templates/job-vault-redis.yaml | 100 + .../prd/erpc/templates/prometheus-rules.yaml | 335 +++ .../prd/erpc/templates/rbac-vault-redis.yaml | 44 + .../erpc/templates/serviceaccount-vault.yaml | 13 + helm/environments/prd/erpc/values.yaml | 1972 +++++++++++++++++ review/repo-map.md | 24 +- validate-helm-charts.sh | 230 ++ 43 files changed, 5013 insertions(+), 11 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/helm-charts-validation.yaml create mode 100644 .github/workflows/slack-pr-notification.yaml create mode 100644 .github/workflows/wiz-iac-scan.yml create mode 100644 helm/charts/.gitkeep create mode 100644 helm/charts/erpc/.env.example create mode 100644 helm/charts/erpc/Chart.yaml create mode 100644 helm/charts/erpc/README.md create mode 100644 helm/charts/erpc/docker-compose.yml create mode 100644 helm/charts/erpc/templates/_helpers.tpl create mode 100644 helm/charts/erpc/templates/deployment.yaml create mode 100644 helm/charts/erpc/templates/hpa.yaml create mode 100644 helm/charts/erpc/templates/image-catalog-pg18.yaml create mode 100644 helm/charts/erpc/templates/job-vault-config.yaml create mode 100644 helm/charts/erpc/templates/job-vault-db.yaml create mode 100644 helm/charts/erpc/templates/monitoring-configmap.yaml create mode 100644 helm/charts/erpc/templates/pdb.yaml create mode 100644 helm/charts/erpc/templates/pg-erpc.yaml create mode 100644 helm/charts/erpc/templates/podmonitor-erpc-db.yaml create mode 100644 helm/charts/erpc/templates/podmonitor.yaml create mode 100644 helm/charts/erpc/templates/rbac-vault.yaml create mode 100644 helm/charts/erpc/templates/scheduled-backup.yaml create mode 100644 helm/charts/erpc/templates/service-headless.yaml create mode 100644 helm/charts/erpc/templates/service-metrics.yaml create mode 100644 helm/charts/erpc/templates/service.yaml create mode 100644 helm/charts/erpc/templates/serviceaccount.yaml create mode 100755 helm/charts/erpc/test-local.sh create mode 100644 helm/charts/erpc/values.yaml create mode 100644 helm/environments/prd/.gitkeep create mode 100644 helm/environments/prd/erpc/Chart.lock create mode 100644 helm/environments/prd/erpc/Chart.yaml create mode 100644 helm/environments/prd/erpc/charts/erpc-0.1.35.tgz create mode 100644 helm/environments/prd/erpc/charts/haproxy-1.26.1.tgz create mode 100644 helm/environments/prd/erpc/charts/prometheus-redis-exporter-6.8.0.tgz create mode 100644 helm/environments/prd/erpc/charts/redis-ha-4.35.2.tgz create mode 100644 helm/environments/prd/erpc/templates/_helpers.tpl create mode 100644 helm/environments/prd/erpc/templates/job-vault-redis.yaml create mode 100644 helm/environments/prd/erpc/templates/prometheus-rules.yaml create mode 100644 helm/environments/prd/erpc/templates/rbac-vault-redis.yaml create mode 100644 helm/environments/prd/erpc/templates/serviceaccount-vault.yaml create mode 100644 helm/environments/prd/erpc/values.yaml create mode 100755 validate-helm-charts.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..d5985d7fd --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# Default ownership +* @morpho-org/platform-engineers + +# Helm charts migrated from morpho-infra-helm (2026-06-17) +/helm/ @morpho-org/platform-engineers @morpho-org/api-engineers +/validate-helm-charts.sh @morpho-org/platform-engineers +/setup-hooks.sh @morpho-org/platform-engineers diff --git a/.github/workflows/helm-charts-validation.yaml b/.github/workflows/helm-charts-validation.yaml new file mode 100644 index 000000000..17b13db14 --- /dev/null +++ b/.github/workflows/helm-charts-validation.yaml @@ -0,0 +1,150 @@ +name: Helm Chart Validation + +on: + push: + branches: [morpho-main] + paths: + - 'helm/**' + pull_request: + branches: [morpho-main] + paths: + - 'helm/**' + +jobs: + # ============================================================================= + # Job 1: Discover all charts to validate + # ============================================================================= + discover-charts: + name: Discover Charts + runs-on: ubuntu-latest + outputs: + charts: ${{ steps.find.outputs.charts }} + chart-count: ${{ steps.find.outputs.count }} + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Find all charts + id: find + run: | + # Find all Chart.yaml files and get their directories + charts=$(find helm/charts helm/environments -name Chart.yaml -type f -exec dirname {} \; | sort | jq -R -s -c 'split("\n") | map(select(length > 0))') + count=$(echo "$charts" | jq 'length') + echo "Found $count charts to validate" + echo "charts=$charts" >> $GITHUB_OUTPUT + echo "count=$count" >> $GITHUB_OUTPUT + + # ============================================================================= + # Job 2: Validate each chart in parallel + # ============================================================================= + validate: + name: Validate + needs: discover-charts + runs-on: ubuntu-latest + strategy: + fail-fast: false # Continue validating other charts if one fails + matrix: + chart: ${{ fromJson(needs.discover-charts.outputs.charts) }} + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up Helm + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4 + with: + version: v3.16.4 + + - name: Install kubeconform + run: | + KUBECONFORM_VERSION=v0.6.1 + KUBECONFORM_SHA256=7f84c1e4109fb72f151da41bdffea05e1e37591f179de51adc048c83533ad215 + curl -fsSLo /tmp/kubeconform.tar.gz "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" + echo "${KUBECONFORM_SHA256} /tmp/kubeconform.tar.gz" | sha256sum -c - + tar xzf /tmp/kubeconform.tar.gz + sudo mv kubeconform /usr/local/bin/ + + - name: Add Helm repositories + run: | + helm repo add bitnami https://charts.bitnami.com/bitnami 2>/dev/null || true + helm repo add cnpg https://cloudnative-pg.github.io/charts 2>/dev/null || true + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 2>/dev/null || true + helm repo add haproxytech https://haproxytech.github.io/helm-charts 2>/dev/null || true + helm repo add dandydeveloper https://dandydeveloper.github.io/charts 2>/dev/null || true + helm repo update + + - name: Get chart info + id: chart-info + run: | + chart_path="${{ matrix.chart }}" + chart_name=$(basename "$chart_path") + echo "name=$chart_name" >> $GITHUB_OUTPUT + echo "path=$chart_path" >> $GITHUB_OUTPUT + + # Check if chart should skip kubeconform validation + skip_validation="false" + case "$chart_name" in + *-db|*database*|*postgres*) + skip_validation="true" + ;; + esac + echo "skip-kubeconform=$skip_validation" >> $GITHUB_OUTPUT + + - name: Helm lint + run: | + echo "::group::Helm lint ${{ steps.chart-info.outputs.name }}" + helm lint "${{ matrix.chart }}" + echo "::endgroup::" + + - name: Update dependencies + run: | + cd "${{ matrix.chart }}" + if grep -q "dependencies:" Chart.yaml 2>/dev/null; then + echo "::group::Updating dependencies for ${{ steps.chart-info.outputs.name }}" + helm dependency update + echo "::endgroup::" + fi + + - name: Kubeconform validation + if: steps.chart-info.outputs.skip-kubeconform != 'true' + run: | + CUSTOM_RESOURCES="PodMonitor,ServiceMonitor,AlertManager,Prometheus,PrometheusRule,Cluster,ScheduledBackup,ImageCatalog,IngressRoute,Certificate,CronJob,EventSource,EventBus,Sensor" + + echo "::group::Kubeconform validation for ${{ steps.chart-info.outputs.name }}" + cd "${{ matrix.chart }}" + helm template . | kubeconform --ignore-missing-schemas --summary --skip "$CUSTOM_RESOURCES" + echo "::endgroup::" + + - name: Skip notice + if: steps.chart-info.outputs.skip-kubeconform == 'true' + run: | + echo "::notice::Skipping kubeconform validation for ${{ steps.chart-info.outputs.name }} (database chart)" + + # ============================================================================= + # Job 3: Summary (ensures all validations completed) + # ============================================================================= + validation-summary: + name: Validation Summary + needs: [discover-charts, validate] + runs-on: ubuntu-latest + if: always() + steps: + - name: Check validation results + run: | + if [ "${{ needs.discover-charts.result }}" == "failure" ]; then + echo "::error::Chart discovery failed" + exit 1 + elif [ "${{ needs.discover-charts.result }}" == "cancelled" ]; then + echo "::warning::Chart discovery was cancelled" + exit 1 + elif [ "${{ needs.validate.result }}" == "failure" ]; then + echo "::error::Some chart validations failed" + exit 1 + elif [ "${{ needs.validate.result }}" == "cancelled" ]; then + echo "::warning::Validation was cancelled" + exit 1 + elif [ "${{ needs.validate.result }}" == "skipped" ]; then + echo "::warning::Validation was skipped unexpectedly" + exit 1 + else + echo "::notice::All ${{ needs.discover-charts.outputs.chart-count }} charts validated successfully" + fi diff --git a/.github/workflows/slack-pr-notification.yaml b/.github/workflows/slack-pr-notification.yaml new file mode 100644 index 000000000..5a439d0fe --- /dev/null +++ b/.github/workflows/slack-pr-notification.yaml @@ -0,0 +1,49 @@ +name: Slack PR Notification + +on: + pull_request: + types: [closed] + branches: [morpho-main] + +jobs: + notify-slack: + if: github.event.pull_request.merged == true && github.event.pull_request.user.login != 'github-actions[bot]' && github.event.pull_request.user.login != 'morpho-infra-deployer[bot]' + runs-on: ubuntu-latest + + steps: + - name: Send Slack notification + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + with: + webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + webhook-type: incoming-webhook + payload: | + { + "text": ":rocket: PR merged to Production", + "attachments": [ + { + "color": "#ff6b6b", + "fallback": "PR #${{ github.event.pull_request.number }} merged: ${{ github.event.pull_request.title }}", + "title": ":white_check_mark: PR #${{ github.event.pull_request.number }}: ${{ github.event.pull_request.title }}", + "title_link": "${{ github.event.pull_request.html_url }}", + "fields": [ + { + "title": ":busts_in_silhouette: Author", + "value": "${{ github.event.pull_request.user.login }}", + "short": true + }, + { + "title": ":earth_americas: Environment", + "value": ":red_circle: Production (morpho-main)", + "short": true + }, + { + "title": ":arrow_right: Branch", + "value": "`${{ github.event.pull_request.head.ref }}` → `${{ github.event.pull_request.base.ref }}`", + "short": false + } + ], + "footer": ":octopus: ${{ github.repository }}", + "footer_icon": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png" + } + ] + } diff --git a/.github/workflows/wiz-iac-scan.yml b/.github/workflows/wiz-iac-scan.yml new file mode 100644 index 000000000..a16ddc108 --- /dev/null +++ b/.github/workflows/wiz-iac-scan.yml @@ -0,0 +1,35 @@ +name: Wiz IaC Scan + +on: + pull_request: + paths: + - 'helm/**' + +permissions: {} + +jobs: + wiz-iac-scan: + name: Wiz IaC Scan + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: arc-morpho-infra-dev + permissions: + contents: read + env: + WIZCLI_VERSION: "1.45.0" + WIZCLI_SHA256: "3f25af9edd63dbd7ecd058b71912f8415467be3f0fcb2b28c6914ebc454d368e" + + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Download Wiz CLI + run: | + curl -fsSLo wizcli "https://downloads.wiz.io/v1/wizcli/${WIZCLI_VERSION}/wizcli-linux-amd64" + echo "${WIZCLI_SHA256} wizcli" | sha256sum -c - + chmod +x wizcli + + - name: Run Wiz CLI IaC scan + env: + WIZ_CLIENT_ID: ${{ secrets.WIZ_CLIENT_ID }} + WIZ_CLIENT_SECRET: ${{ secrets.WIZ_CLIENT_SECRET }} + run: ./wizcli scan dir helm/ diff --git a/helm/charts/.gitkeep b/helm/charts/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/helm/charts/erpc/.env.example b/helm/charts/erpc/.env.example new file mode 100644 index 000000000..c6389d481 --- /dev/null +++ b/helm/charts/erpc/.env.example @@ -0,0 +1,23 @@ +# eRPC Environment Configuration +# Copy this file to .env and fill in your actual API keys + +# Provider API Keys +ALCHEMY_API_KEY=your-alchemy-api-key-here +DRPC_API_KEY=your-drpc-api-key-here +TENDERLY_API_KEY=your-tenderly-api-key-here +TENDERLY_PROJECT=your-tenderly-project-slug +ERPC_API_KEY=your-erpc-api-key-here + +# Authentication +LABELS_SERVICE_API_KEY=labels-service-api-key-here +SQD_INDEXERS_API_KEY=sqd-indexers-api-key-here +BLUE_API_API_KEY=blue-api-key-here + +# Optional Configuration +LOG_LEVEL=info + +# Instructions: +# 1. Copy this file: cp .env.example .env +# 2. Edit .env with your actual API keys +# 3. Run: docker-compose up -d +# 4. Test: ./test-local.sh \ No newline at end of file diff --git a/helm/charts/erpc/Chart.yaml b/helm/charts/erpc/Chart.yaml new file mode 100644 index 000000000..ecc04b052 --- /dev/null +++ b/helm/charts/erpc/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: erpc +description: eRPC fault-tolerant EVM RPC proxy + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.35 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "latest" diff --git a/helm/charts/erpc/README.md b/helm/charts/erpc/README.md new file mode 100644 index 000000000..182999157 --- /dev/null +++ b/helm/charts/erpc/README.md @@ -0,0 +1,61 @@ +# eRPC Helm Chart + +This Helm chart deploys eRPC as a fault-tolerant EVM RPC proxy service within the Morpho infrastructure. + +## Local Development with Docker Compose + +For local testing and development: + +```bash +# Navigate to chart directory +cd charts/erpc/ + +# Copy environment template and edit with your API keys +cp .env.example .env +# Edit .env with your actual API keys and AUTH_SECRET + +# Start services (Redis + eRPC) +docker-compose up -d + +# Run tests +./test-local.sh + +# Stop services +docker-compose down +``` + + +## Usage + +When authentication is enabled, requests must include the secret as a query parameter: + +```bash +curl --location 'http://localhost:4000/cache/evm/1?secret=YOUR-AUTH-SECRET' \ + --header 'Content-Type: application/json' \ + --data '{"method":"eth_blockNumber","params":[],"id":1,"jsonrpc":"2.0"}' +``` + +## Vault Config + +The eRPC chart reads non-secret config from `vault.configPath` and secret values +from `vault.secretsPath`. Keep `vault.configPath` as a config template. Static +secret placeholders use `__SECRET___`; the API-key auth list can use the +`__SECRET_API_KEY_STRATEGIES__` marker to generate one secret auth strategy for +every `API_KEY_*=` entry in `vault.secretsPath`. Generated auth IDs are +derived from the key name after `API_KEY_`, lowercased with `_` converted to `-`; +trailing hash rotation suffixes like `_9E41B694` are ignored so rotating a value +does not change the eRPC secret ID. Optional per-key metadata entries can tune +generated strategies without creating new API keys: `API_KEY_FOO__ID` pins the +eRPC secret ID, `API_KEY_FOO__ORDER` pins list order, `API_KEY_FOO__ALLOW_METHODS` +is a comma-separated method allowlist, and `API_KEY_FOO__RATE_LIMIT_BUDGET` sets +`rateLimitBudget`. + +Vault secret keys must match `[A-Za-z_][A-Za-z0-9_]*`. Secret values are parsed +from a line-oriented `KEY=VALUE` file, so keep them single-line. Placeholders in +quoted YAML scalars are YAML-escaped by the renderer; placeholders embedded in +unquoted YAML scalars, such as URL path fragments, must have YAML-plain-safe +values. + +The Vault config job renders placeholders into a temporary `erpc.yaml`, validates +that rendered config with `erpc validate`, then stores only the rendered file in +the runtime Kubernetes Secret. diff --git a/helm/charts/erpc/docker-compose.yml b/helm/charts/erpc/docker-compose.yml new file mode 100644 index 000000000..09decb3af --- /dev/null +++ b/helm/charts/erpc/docker-compose.yml @@ -0,0 +1,14 @@ +services: + erpc: + image: ghcr.io/erpc/erpc:latest + ports: + - "4000:4000" # HTTP RPC port + - "4001:4001" # Metrics port + volumes: + - ./config/erpc.yaml:/erpc.yaml:ro + env_file: + - .env + environment: + # DynamoDB configuration uses AWS credentials from environment or IAM + AWS_REGION: "eu-west-3" + restart: unless-stopped \ No newline at end of file diff --git a/helm/charts/erpc/templates/_helpers.tpl b/helm/charts/erpc/templates/_helpers.tpl new file mode 100644 index 000000000..2b72a303d --- /dev/null +++ b/helm/charts/erpc/templates/_helpers.tpl @@ -0,0 +1,87 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "erpc.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "erpc.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "erpc.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "erpc.labels" -}} +helm.sh/chart: {{ include "erpc.chart" . }} +{{ include "erpc.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "erpc.selectorLabels" -}} +app.kubernetes.io/name: {{ include "erpc.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Vault creator ServiceAccount name. +*/}} +{{- define "erpc.vaultCreatorServiceAccountName" -}} +{{- $vaultCreator := .Values.serviceAccount.vaultCreator | default dict -}} +{{- $vaultCreator.name | default .Values.serviceAccount.name | default "erpc" }} +{{- end }} + +{{/* +Runtime ServiceAccount name. +*/}} +{{- define "erpc.serviceAccountName" -}} +{{- if .Values.serviceAccount.runtimeName -}} +{{- .Values.serviceAccount.runtimeName -}} +{{- else if eq (toString .Values.serviceAccount.create) "false" -}} +{{- include "erpc.vaultCreatorServiceAccountName" . -}} +{{- else -}} +{{- printf "%s-runtime" (include "erpc.vaultCreatorServiceAccountName" .) -}} +{{- end -}} +{{- end }} + +{{/* +Kubernetes Secrets managed by Vault creator jobs. +*/}} +{{- define "erpc.vaultSecretResourceNames" -}} +- "erpc-db-secret" +- {{ printf "%s-vault-config" (include "erpc.fullname" .) | quote }} +{{- end }} + +{{/* +Common environment variables for erpc +*/}} +{{- define "erpc.commonEnv" -}} +# All env vars now come from vault-secrets via envFrom +{{- end -}} diff --git a/helm/charts/erpc/templates/deployment.yaml b/helm/charts/erpc/templates/deployment.yaml new file mode 100644 index 000000000..0044065b3 --- /dev/null +++ b/helm/charts/erpc/templates/deployment.yaml @@ -0,0 +1,139 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "erpc.fullname" . }} + annotations: + reloader.stakater.com/auto: "true" + labels: + {{- include "erpc.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + strategy: + rollingUpdate: + # Let the new ReplicaSet surge relative to the current HPA target, + # not just minReplicas, so rollouts can warm a full replacement pool. + maxSurge: 100% + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + {{- include "erpc.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "erpc.selectorLabels" . | nindent 8 }} + # Environment variables loaded from vault secrets via pre-install job + spec: + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 45 }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "erpc.serviceAccountName" . }} + securityContext: + {{- if .Values.securityHardening.enabled }} + {{- toYaml .Values.securityHardening.podSecurityContext | nindent 8 }} + {{- else }} + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + {{- include "erpc.selectorLabels" . | nindent 14 }} + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + {{- include "erpc.selectorLabels" . | nindent 14 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- if .Values.securityHardening.enabled }} + {{- toYaml .Values.securityHardening.containerSecurityContext | nindent 12 }} + {{- else }} + {{- toYaml .Values.securityContext | nindent 12 }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.config.server.httpPort }} + protocol: TCP + - name: metrics + containerPort: {{ .Values.config.server.metricsPort }} + protocol: TCP + env: + {{- with .Values.extraEnvVars }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /erpc.yaml + subPath: erpc.yaml + readOnly: true + {{- if .Values.securityHardening.enabled }} + - name: tmp + mountPath: /tmp + {{- end }} + startupProbe: + httpGet: + path: {{ .Values.startupProbe.path | default "/healthcheck" }} + port: http + initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds | default 10 }} + periodSeconds: {{ .Values.startupProbe.periodSeconds | default 5 }} + timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds | default 5 }} + failureThreshold: {{ .Values.startupProbe.failureThreshold | default 6 }} + livenessProbe: + tcpSocket: + port: http + {{- if .Values.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + {{- end }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds | default 10 }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds | default 5 }} + successThreshold: 1 + failureThreshold: {{ .Values.livenessProbe.failureThreshold | default 3 }} + readinessProbe: + httpGet: + path: {{ .Values.readinessProbe.path | default "/healthcheck" }} + port: http + {{- if .Values.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + {{- end }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds | default 5 }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds | default 30 }} + successThreshold: 1 + failureThreshold: {{ .Values.readinessProbe.failureThreshold | default 2 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- if .Values.lifecycle.enabled }} + lifecycle: + {{- omit .Values.lifecycle "enabled" | toYaml | nindent 12 }} + {{- end }} + volumes: + - name: config + secret: + secretName: {{ include "erpc.fullname" . }}-vault-config + optional: true + {{- if .Values.securityHardening.enabled }} + - name: tmp + emptyDir: {} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/charts/erpc/templates/hpa.yaml b/helm/charts/erpc/templates/hpa.yaml new file mode 100644 index 000000000..b65a1cfb8 --- /dev/null +++ b/helm/charts/erpc/templates/hpa.yaml @@ -0,0 +1,38 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "erpc.fullname" . }} + labels: + {{- include "erpc.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "erpc.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Pods + pods: + metric: + name: cpu_utilization_percentage + target: + type: AverageValue + averageValue: "{{ .Values.autoscaling.targetCPUUtilizationPercentage }}" + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Pods + pods: + metric: + name: memory_utilization_percentage + target: + type: AverageValue + averageValue: "{{ .Values.autoscaling.targetMemoryUtilizationPercentage }}" + {{- end }} + {{- with .Values.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/charts/erpc/templates/image-catalog-pg18.yaml b/helm/charts/erpc/templates/image-catalog-pg18.yaml new file mode 100644 index 000000000..d265b7149 --- /dev/null +++ b/helm/charts/erpc/templates/image-catalog-pg18.yaml @@ -0,0 +1,12 @@ +{{- if ne (toString .Values.postgresql.enabled) "false" }} +apiVersion: postgresql.cnpg.io/v1 +kind: ImageCatalog +metadata: + name: erpc-postgresql-native-18 + annotations: + argocd.argoproj.io/sync-wave: "-3" +spec: + images: + - major: 18 + image: "ghcr.io/cloudnative-pg/postgresql:18.0-system-trixie" +{{- end }} diff --git a/helm/charts/erpc/templates/job-vault-config.yaml b/helm/charts/erpc/templates/job-vault-config.yaml new file mode 100644 index 000000000..bf1d58ada --- /dev/null +++ b/helm/charts/erpc/templates/job-vault-config.yaml @@ -0,0 +1,444 @@ +{{- $validationImage := .Values.vault.validationImage | default dict }} +{{- if and $validationImage.enabled (or (not $validationImage.repository) (not $validationImage.tag)) }} +{{- fail "vault.validationImage.repository and vault.validationImage.tag are required when vault.validationImage.enabled=true" }} +{{- end }} +{{- $useDedicatedValidator := $validationImage.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "erpc.fullname" . }}-vault-config-creator + labels: + {{- include "erpc.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded + argocd.argoproj.io/sync-wave: "-3" + argocd.argoproj.io/sync-options: Force=true,Replace=true +spec: + template: + metadata: + labels: + {{- include "erpc.selectorLabels" . | nindent 8 }} + job: vault-config-creator + annotations: + # Vault Agent annotations for config injection (on pod template) + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: {{ include "erpc.vaultCreatorServiceAccountName" . | quote }} + vault.hashicorp.com/agent-inject-secret-erpc-config: {{ .Values.vault.configPath | default "secret/data/erpc/config" | quote }} + vault.hashicorp.com/agent-inject-secret-erpc-secrets: {{ .Values.vault.secretsPath | default "secret/data/erpc/secrets" | quote }} + vault.hashicorp.com/agent-inject-template-erpc-config: | + {{`{{- with secret "`}}{{ .Values.vault.configPath | default "secret/data/erpc/config" }}{{`" -}} + {{- .Data.data.config -}} + {{- end }}`}} + vault.hashicorp.com/agent-inject-template-erpc-secrets: | + {{`{{- with secret "`}}{{ .Values.vault.secretsPath | default "secret/data/erpc/secrets" }}{{`" -}} + {{- range $key, $value := .Data.data -}} + {{ printf "%s=%s\n" $key $value -}} + {{- end -}} + {{- end }}`}} + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + spec: + serviceAccountName: {{ include "erpc.vaultCreatorServiceAccountName" . }} + automountServiceAccountToken: true + {{- if .Values.securityHardening.enabled }} + securityContext: + {{- toYaml .Values.securityHardening.podSecurityContext | nindent 8 }} + {{- end }} + restartPolicy: OnFailure + volumes: + - name: rendered-config + emptyDir: {} + {{- if .Values.securityHardening.enabled }} + - name: tmp + emptyDir: {} + {{- end }} + initContainers: + - name: config-renderer + {{- if .Values.securityHardening.enabled }} + securityContext: + {{- toYaml .Values.securityHardening.containerSecurityContext | nindent 12 }} + {{- end }} + image: "{{ .Values.vault.jobImage.repository }}:{{ .Values.vault.jobImage.tag }}" + imagePullPolicy: {{ .Values.vault.jobImage.pullPolicy }} + resources: + {{- toYaml .Values.vault.jobResources | nindent 12 }} + {{- if .Values.securityHardening.enabled }} + env: + - name: HOME + value: /tmp + {{- end }} + volumeMounts: + - name: rendered-config + mountPath: /work + {{- if .Values.securityHardening.enabled }} + - name: tmp + mountPath: /tmp + {{- end }} + command: + - sh + - -c + - | + set -eu + if (set -o pipefail) 2>/dev/null; then + set -o pipefail + fi + + echo "Rendering ERPC configuration from vault config and secrets..." + + timeout=60 + count=0 + while { [ ! -f /vault/secrets/erpc-config ] || [ ! -f /vault/secrets/erpc-secrets ]; } && [ $count -lt $timeout ]; do + echo "Waiting for vault config and secrets files... ($count/$timeout)" + sleep 2 + count=$((count + 2)) + done + + if [ ! -f /vault/secrets/erpc-config ]; then + echo "ERROR: Vault config file not found after $timeout seconds" + exit 1 + fi + + if [ ! -f /vault/secrets/erpc-secrets ]; then + echo "ERROR: Vault secrets file not found after $timeout seconds" + exit 1 + fi + + tmp_rendered_config=/tmp/erpc-rendered-config + rendered_config=/work/erpc.yaml + trap 'rm -f "$tmp_rendered_config"' EXIT + + awk -v secrets_file="/vault/secrets/erpc-secrets" ' + function yaml_quote(value, quoted) { + quoted = value + gsub(/\\/, "\\\\", quoted) + gsub(/"/, "\\\"", quoted) + gsub(/\r/, "\\r", quoted) + gsub(/\n/, "\\n", quoted) + gsub(/\t/, "\\t", quoted) + return "\"" quoted "\"" + } + + function metadata_key(key, suffix) { + return key "__" suffix + } + + function api_key_base(key, suffix) { + suffix = substr(key, length(key) - 8) + if (suffix ~ /^_[0-9A-Fa-f]{8}$/) { + return substr(key, 1, length(key) - 9) + } + return key + } + + function metadata_lookup_key(key, suffix, exact_key, base_key) { + exact_key = metadata_key(key, suffix) + if (exact_key in secrets) { + return exact_key + } + + base_key = metadata_key(api_key_base(key), suffix) + if (base_key in secrets) { + return base_key + } + + return "" + } + + function api_key_id(key, id, id_key) { + id_key = metadata_lookup_key(key, "ID") + if (id_key != "") { + return secrets[id_key] + } + + id = substr(api_key_base(key), 9) + id = tolower(id) + gsub(/_+/, "-", id) + return id + } + + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + + function emit_method_list(indent, methods, parts, count, i, method) { + count = split(methods, parts, ",") + for (i = 1; i <= count; i++) { + method = trim(parts[i]) + if (method != "") { + print indent " - " yaml_quote(method) + } + } + } + + function api_key_sort_value(key, order_key) { + order_key = metadata_lookup_key(key, "ORDER") + if (order_key != "" && secrets[order_key] ~ /^[0-9]+$/) { + return sprintf("%09d:%s", secrets[order_key] + 0, key) + } + return "999999999:" key + } + + function sort_api_keys(i, j, tmp, left, right) { + if (api_key_order_sorted) { + return + } + + for (i = 1; i <= api_key_count; i++) { + for (j = i + 1; j <= api_key_count; j++) { + left = api_key_sort_value(api_key_order[i]) + right = api_key_sort_value(api_key_order[j]) + if (right < left) { + tmp = api_key_order[i] + api_key_order[i] = api_key_order[j] + api_key_order[j] = tmp + } + } + } + api_key_order_sorted = 1 + } + + function emit_api_key_strategies(indent, i, key, allow_key, budget_key) { + if (api_key_count == 0) { + print "ERROR: __SECRET_API_KEY_STRATEGIES__ used but no API_KEY_* entries exist in Vault secrets" > "/dev/stderr" + exit 1 + } + + sort_api_keys() + for (i = 1; i <= api_key_count; i++) { + key = api_key_order[i] + allow_key = metadata_lookup_key(key, "ALLOW_METHODS") + budget_key = metadata_lookup_key(key, "RATE_LIMIT_BUDGET") + + print indent "- type: secret" + if (budget_key != "") { + print indent " rateLimitBudget: " yaml_quote(secrets[budget_key]) + } + if (allow_key != "") { + print indent " ignoreMethods:" + print indent " - " yaml_quote("*") + } + print indent " secret:" + print indent " id: " yaml_quote(api_key_id(key)) + print indent " value: " yaml_quote(secrets[key]) + if (allow_key != "") { + print indent " allowMethods:" + emit_method_list(indent, secrets[allow_key]) + } + } + } + + BEGIN { + missing_count = 0 + invalid_count = 0 + api_key_count = 0 + api_key_order_sorted = 0 + while ((getline line < secrets_file) > 0) { + if (line == "") { + continue + } + + delimiter = index(line, "=") + if (delimiter == 0) { + print "ERROR: Invalid Vault secret entry without = delimiter" > "/dev/stderr" + invalid_count++ + continue + } + + key = substr(line, 1, delimiter - 1) + value = substr(line, delimiter + 1) + + if (key !~ /^[A-Za-z_][A-Za-z0-9_]*$/) { + print "ERROR: Invalid Vault secret key " key ". Use [A-Za-z_][A-Za-z0-9_]*." > "/dev/stderr" + invalid_count++ + continue + } + + secrets[key] = value + if (key ~ /^API_KEY_/ && key !~ /__(ALLOW_METHODS|RATE_LIMIT_BUDGET|ID|ORDER)$/) { + api_key_order[++api_key_count] = key + } + } + close(secrets_file) + + if (invalid_count > 0) { + exit 1 + } + } + { + while (match($0, /"__SECRET_[A-Za-z_][A-Za-z0-9_]*__"/)) { + token = substr($0, RSTART + 1, RLENGTH - 2) + key = substr(token, 10, length(token) - 11) + + if (key in secrets) { + repl = yaml_quote(secrets[key]) + } else { + repl = yaml_quote("") + if (!(token in missing)) { + missing[token] = 1 + missing_order[++missing_count] = token + } + } + + $0 = substr($0, 1, RSTART - 1) repl substr($0, RSTART + RLENGTH) + } + + while (match($0, /'\''__SECRET_[A-Za-z_][A-Za-z0-9_]*__'\''/)) { + token = substr($0, RSTART + 1, RLENGTH - 2) + key = substr(token, 10, length(token) - 11) + + if (key in secrets) { + repl = yaml_quote(secrets[key]) + } else { + repl = yaml_quote("") + if (!(token in missing)) { + missing[token] = 1 + missing_order[++missing_count] = token + } + } + + $0 = substr($0, 1, RSTART - 1) repl substr($0, RSTART + RLENGTH) + } + + if (match($0, /^[[:space:]]*-[[:space:]]*__SECRET_API_KEY_STRATEGIES__[[:space:]]*$/)) { + indent = substr($0, 1, index($0, "-") - 1) + emit_api_key_strategies(indent) + next + } + + if (match($0, /^[[:space:]]*__SECRET_API_KEY_STRATEGIES__[[:space:]]*$/)) { + indent = $0 + sub(/__SECRET_API_KEY_STRATEGIES__.*/, "", indent) + emit_api_key_strategies(indent) + next + } + + out = "" + rest = $0 + while (match(rest, /__SECRET_[A-Za-z_][A-Za-z0-9_]*__/)) { + token = substr(rest, RSTART, RLENGTH) + key = substr(token, 10, length(token) - 11) + + if (key in secrets) { + repl = secrets[key] + } else { + repl = token + if (!(token in missing)) { + missing[token] = 1 + missing_order[++missing_count] = token + } + } + + out = out substr(rest, 1, RSTART - 1) repl + rest = substr(rest, RSTART + RLENGTH) + } + print out rest + } + END { + if (missing_count > 0) { + print "ERROR: Unresolved ERPC secret placeholders remain in rendered config:" > "/dev/stderr" + for (i = 1; i <= missing_count; i++) { + print missing_order[i] > "/dev/stderr" + } + exit 1 + } + } + ' /vault/secrets/erpc-config > "$tmp_rendered_config" + mv "$tmp_rendered_config" "$rendered_config" + + echo "ERPC configuration rendered to $rendered_config" + {{- if $useDedicatedValidator }} + - name: config-validator + {{- if .Values.securityHardening.enabled }} + securityContext: + {{- toYaml .Values.securityHardening.containerSecurityContext | nindent 12 }} + {{- end }} + image: "{{ $validationImage.repository }}:{{ $validationImage.tag }}" + imagePullPolicy: {{ $validationImage.pullPolicy | default .Values.vault.jobImage.pullPolicy }} + {{- if .Values.securityHardening.enabled }} + env: + - name: HOME + value: /tmp + {{- end }} + volumeMounts: + - name: rendered-config + mountPath: /work + {{- if .Values.securityHardening.enabled }} + - name: tmp + mountPath: /tmp + {{- end }} + command: + {{- toYaml ($validationImage.command | default (list "erpc")) | nindent 12 }} + args: + - validate + - --config + - /work/erpc.yaml + resources: + {{- toYaml ($validationImage.resources | default .Values.vault.jobResources) | nindent 12 }} + {{- end }} + containers: + - name: config-creator + {{- if .Values.securityHardening.enabled }} + securityContext: + {{- toYaml .Values.securityHardening.containerSecurityContext | nindent 12 }} + {{- end }} + image: "{{ .Values.vault.jobImage.repository }}:{{ .Values.vault.jobImage.tag }}" + imagePullPolicy: {{ .Values.vault.jobImage.pullPolicy }} + resources: + {{- toYaml .Values.vault.jobResources | nindent 12 }} + {{- if .Values.securityHardening.enabled }} + env: + - name: HOME + value: /tmp + {{- end }} + volumeMounts: + - name: rendered-config + mountPath: /work + {{- if .Values.securityHardening.enabled }} + - name: tmp + mountPath: /tmp + {{- end }} + command: + - sh + - -c + - | + set -eu + if (set -o pipefail) 2>/dev/null; then + set -o pipefail + fi + + rendered_config=/work/erpc.yaml + echo "Creating Kubernetes Secret from rendered vault config..." + + if [ ! -s "$rendered_config" ]; then + echo "ERROR: Rendered ERPC config file not found or empty: $rendered_config" + exit 1 + fi + + {{- if not $useDedicatedValidator }} + echo "Validating ERPC configuration..." + + # Validate ERPC configuration using erpc binary + if ! erpc validate --config "$rendered_config"; then + echo "ERROR: Invalid ERPC configuration" + exit 1 + fi + + echo "ERPC configuration validation passed. Creating Kubernetes Secret..." + {{- else }} + echo "ERPC configuration validation passed with dedicated runtime image. Creating Kubernetes Secret..." + {{- end }} + + # Create secret from rendered vault config file + kubectl create secret generic {{ include "erpc.fullname" . }}-vault-config \ + --namespace={{ .Release.Namespace }} \ + --from-file=erpc.yaml="$rendered_config" \ + --dry-run=client -o yaml | kubectl apply -f - + + echo "Secret {{ include "erpc.fullname" . }}-vault-config created successfully" + + # Verify secret was created + kubectl get secret {{ include "erpc.fullname" . }}-vault-config -n {{ .Release.Namespace }} diff --git a/helm/charts/erpc/templates/job-vault-db.yaml b/helm/charts/erpc/templates/job-vault-db.yaml new file mode 100644 index 000000000..8f45994b6 --- /dev/null +++ b/helm/charts/erpc/templates/job-vault-db.yaml @@ -0,0 +1,89 @@ +{{- if ne (toString .Values.postgresql.enabled) "false" }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "erpc.fullname" . }}-db-secret-creator + labels: + {{- include "erpc.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded + argocd.argoproj.io/sync-wave: "-3" + argocd.argoproj.io/sync-options: Force=true,Replace=true +spec: + template: + metadata: + labels: + {{- include "erpc.selectorLabels" . | nindent 8 }} + job: db-secret-creator + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/role: {{ include "erpc.vaultCreatorServiceAccountName" . | quote }} + vault.hashicorp.com/agent-inject-secret-secrets: {{ .Values.postgresql.vault.secretsPath | default "secret/data/erpc/db" | quote }} + vault.hashicorp.com/agent-inject-template-secrets: | + {{`{{- with secret "`}}{{ .Values.postgresql.vault.secretsPath | default "secret/data/erpc/db" }}{{`" -}} + {{- .Data.data.password -}} + {{- end }}`}} + vault.hashicorp.com/agent-pre-populate-only: "true" + spec: + serviceAccountName: {{ include "erpc.vaultCreatorServiceAccountName" . }} + automountServiceAccountToken: true + {{- if .Values.securityHardening.enabled }} + securityContext: + {{- toYaml .Values.securityHardening.podSecurityContext | nindent 8 }} + {{- end }} + restartPolicy: OnFailure + {{- if .Values.securityHardening.enabled }} + volumes: + - name: tmp + emptyDir: {} + {{- end }} + containers: + - name: db-secret-creator + {{- if .Values.securityHardening.enabled }} + securityContext: + {{- toYaml .Values.securityHardening.containerSecurityContext | nindent 12 }} + {{- end }} + image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/alpine/k8s:1.31.0 + {{- if .Values.securityHardening.enabled }} + env: + - name: HOME + value: /tmp + volumeMounts: + - name: tmp + mountPath: /tmp + {{- end }} + command: + - sh + - -c + - | + echo "Creating ERPC database Kubernetes Secret from Vault..." + + timeout=60 + count=0 + while [ ! -f /vault/secrets/secrets ] && [ $count -lt $timeout ]; do + echo "Waiting for vault secrets file... ($count/$timeout)" + sleep 2 + count=$((count + 2)) + done + + if [ ! -f /vault/secrets/secrets ]; then + echo "ERROR: Vault secrets file not found after $timeout seconds" + exit 1 + fi + + echo "Vault secrets file found. Creating Kubernetes Secret..." + + DB_PASSWORD=$(cat /vault/secrets/secrets) + + kubectl create secret generic erpc-db-secret \ + --namespace={{ .Release.Namespace }} \ + --from-literal=username=postgres \ + --from-literal=password="$DB_PASSWORD" \ + --dry-run=client -o yaml | kubectl apply -f - + + echo "Secret erpc-db-secret created successfully" + + kubectl get secret erpc-db-secret -n {{ .Release.Namespace }} +{{- end }} diff --git a/helm/charts/erpc/templates/monitoring-configmap.yaml b/helm/charts/erpc/templates/monitoring-configmap.yaml new file mode 100644 index 000000000..45356facc --- /dev/null +++ b/helm/charts/erpc/templates/monitoring-configmap.yaml @@ -0,0 +1,59 @@ +{{- if ne (toString .Values.postgresql.enabled) "false" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "erpc.fullname" . }}-cnpg-recovery-health-monitoring + labels: + cnpg.io/reload: "" + {{- include "erpc.labels" . | nindent 4 }} +data: + custom-queries: | + # PostgreSQL Recovery Health Metrics + # Detects conditions that prevent hot standby or cause recovery issues + + pg_recovery_health: + query: | + SELECT + CASE WHEN pg_is_in_recovery() THEN 1 ELSE 0 END AS is_replica, + CASE WHEN pg_is_in_recovery() AND NOT pg_is_wal_replay_paused() THEN 1 ELSE 0 END AS recovery_active, + CASE WHEN pg_is_in_recovery() AND pg_is_wal_replay_paused() THEN 1 ELSE 0 END AS recovery_paused, + COALESCE(EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::bigint, 0) AS replication_lag_seconds + metrics: + - is_replica: + usage: "GAUGE" + description: "1 if this instance is a replica, 0 if primary" + - recovery_active: + usage: "GAUGE" + description: "1 if recovery is actively replaying WAL, 0 otherwise" + - recovery_paused: + usage: "GAUGE" + description: "1 if recovery is paused (critical - server will shut down if unpaused)" + - replication_lag_seconds: + usage: "GAUGE" + description: "Seconds since last replayed transaction" + + pg_hot_standby_params: + query: | + SELECT + current_setting('max_connections')::int AS max_connections, + current_setting('max_worker_processes')::int AS max_worker_processes, + current_setting('max_wal_senders')::int AS max_wal_senders, + current_setting('max_prepared_transactions')::int AS max_prepared_transactions, + current_setting('max_locks_per_transaction')::int AS max_locks_per_transaction + metrics: + - max_connections: + usage: "GAUGE" + description: "Current max_connections setting" + - max_worker_processes: + usage: "GAUGE" + description: "Current max_worker_processes setting" + - max_wal_senders: + usage: "GAUGE" + description: "Current max_wal_senders setting" + - max_prepared_transactions: + usage: "GAUGE" + description: "Current max_prepared_transactions setting" + - max_locks_per_transaction: + usage: "GAUGE" + description: "Current max_locks_per_transaction setting" +{{- end }} diff --git a/helm/charts/erpc/templates/pdb.yaml b/helm/charts/erpc/templates/pdb.yaml new file mode 100644 index 000000000..8f9a08f17 --- /dev/null +++ b/helm/charts/erpc/templates/pdb.yaml @@ -0,0 +1,11 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "erpc.fullname" . }}-pdb + labels: + {{- include "erpc.labels" . | nindent 4 }} +spec: + minAvailable: {{ sub .Values.autoscaling.minReplicas 1 }} + selector: + matchLabels: + {{- include "erpc.selectorLabels" . | nindent 6 }} diff --git a/helm/charts/erpc/templates/pg-erpc.yaml b/helm/charts/erpc/templates/pg-erpc.yaml new file mode 100644 index 000000000..e86f43667 --- /dev/null +++ b/helm/charts/erpc/templates/pg-erpc.yaml @@ -0,0 +1,186 @@ +{{- if ne (toString .Values.postgresql.enabled) "false" }} +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: erpc-db + annotations: + argocd.argoproj.io/sync-wave: "-2" +spec: + instances: 2 + primaryUpdateStrategy: unsupervised + imageCatalogRef: + apiGroup: postgresql.cnpg.io + kind: ImageCatalog + name: erpc-postgresql-native-18 + major: 18 + + serviceAccountTemplate: + metadata: + annotations: + eks.amazonaws.com/role-arn: {{ .Values.postgresql.s3SARoleArn }} + postgresql: + shared_preload_libraries: + - pg_stat_statements + parameters: + max_connections: "5000" + + idle_in_transaction_session_timeout: "30000" + + max_locks_per_transaction: "512" + max_pred_locks_per_transaction: "512" + + shared_buffers: {{ .Values.postgresql.sharedBuffers | default "1GB" | quote }} + effective_cache_size: {{ .Values.postgresql.effectiveCacheSize | default "3GB" | quote }} + maintenance_work_mem: {{ .Values.postgresql.maintenanceWorkMem | default "256MB" | quote }} + work_mem: {{ .Values.postgresql.workMem | default "8MB" | quote }} + + wal_buffers: "16MB" + min_wal_size: "1GB" + max_wal_size: "8GB" + wal_keep_size: "2GB" + checkpoint_completion_target: "0.9" + checkpoint_timeout: "15min" + wal_level: "replica" + synchronous_commit: "off" + + default_statistics_target: "200" + random_page_cost: "1.0" + effective_io_concurrency: "200" + huge_pages: "off" + + max_worker_processes: {{ .Values.postgresql.maxWorkerProcesses | default "2" | quote }} + max_parallel_workers: {{ .Values.postgresql.maxParallelWorkers | default "2" | quote }} + max_parallel_workers_per_gather: {{ .Values.postgresql.maxParallelWorkersPerGather | default "1" | quote }} + max_parallel_maintenance_workers: {{ .Values.postgresql.maxParallelMaintenanceWorkers | default "1" | quote }} + + autovacuum_max_workers: {{ .Values.postgresql.autovacuumMaxWorkers | default "2" | quote }} + autovacuum_naptime: "10s" + autovacuum_vacuum_scale_factor: "0.01" + autovacuum_analyze_scale_factor: "0.005" + autovacuum_vacuum_cost_delay: "2ms" + autovacuum_vacuum_cost_limit: "2000" + + track_io_timing: "on" + jit: "off" + + bgwriter_delay: "200ms" + bgwriter_lru_maxpages: "1000" + bgwriter_lru_multiplier: "10.0" + + track_activity_query_size: "8192" + pg_stat_statements.max: "5000" + pg_stat_statements.track: "all" + + # Must exceed worst-case pg_backup_start() duration (checkpoint + WAL flush) + # on a write-heavy DB; 10s was killing every barman base backup. + statement_timeout: "600000" + lock_timeout: "5000" + + vacuum_cost_delay: "2ms" + + # Write-heavy workload optimizations + full_page_writes: "on" + wal_compression: "on" + wal_writer_delay: "10ms" + commit_delay: "10" + commit_siblings: "10" + + # Reduce checkpoint frequency impact + checkpoint_warning: "300s" + + # Connection pooling optimization + tcp_keepalives_idle: "60" + tcp_keepalives_interval: "10" + tcp_keepalives_count: "6" + pg_hba: + - host all all 0.0.0.0/0 md5 + + bootstrap: + initdb: + database: erpc + owner: postgres + secret: + name: erpc-db-secret + postInitApplicationSQL: + - CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + + storage: + storageClass: {{ .Values.postgresql.storageClass | default "gp3" }} + size: {{ .Values.postgresql.storage | default "100Gi" }} + {{- with .Values.postgresql.volumeAttributesClassName }} + pvcTemplate: + volumeAttributesClassName: {{ . | quote }} + {{- end }} + + resources: + {{- toYaml .Values.postgresql.resources | nindent 4 }} + + {{- $aff := .Values.postgresql.affinity | default dict -}} + {{- if or $aff.nodeSelector $aff.tolerations $aff.additionalPodAntiAffinity }} + affinity: + {{- with $aff.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with $aff.tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with $aff.additionalPodAntiAffinity }} + additionalPodAntiAffinity: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- else }} + affinity: {} + {{- end }} + + enableSuperuserAccess: true + superuserSecret: + name: erpc-db-secret + + monitoring: + customQueriesConfigMap: + - key: queries + name: cnpg-default-monitoring + - key: custom-queries + name: {{ include "erpc.fullname" . }}-cnpg-recovery-health-monitoring + + {{- $snapshotClass := .Values.postgresql.backup.volumeSnapshotClassName | default "" }} + {{- $backupMethod := .Values.postgresql.backup.method | default (ternary "volumeSnapshot" "barmanObjectStore" (ne $snapshotClass "")) }} + {{- if eq $backupMethod "volumeSnapshot" }} + {{- if not $snapshotClass }}{{ fail "postgresql.backup.volumeSnapshotClassName is required when postgresql.backup.method=volumeSnapshot" }}{{ end }} + backup: + retentionPolicy: {{ .Values.postgresql.backup.retentionPolicy | default "7d" | quote }} + {{- with .Values.postgresql.backup.destinationPath }} + barmanObjectStore: + s3Credentials: + inheritFromIAMRole: true + destinationPath: {{ . }} + endpointURL: "https://s3.eu-west-3.amazonaws.com" + wal: + compression: gzip + maxParallel: 4 + {{- end }} + target: prefer-standby + volumeSnapshot: + className: {{ $snapshotClass }} + online: true + onlineConfiguration: + immediateCheckpoint: false + waitForArchive: true + snapshotOwnerReference: backup + {{- else if eq $backupMethod "barmanObjectStore" }} + backup: + retentionPolicy: {{ .Values.postgresql.backup.retentionPolicy | default "7d" | quote }} + barmanObjectStore: + s3Credentials: + inheritFromIAMRole: true + destinationPath: {{ .Values.postgresql.backup.destinationPath }} + endpointURL: "https://s3.eu-west-3.amazonaws.com" + wal: + compression: gzip + maxParallel: 4 + {{- else }} + {{- fail (printf "postgresql.backup.method must be \"barmanObjectStore\" or \"volumeSnapshot\", got %q" $backupMethod) }} + {{- end }} +{{- end }} diff --git a/helm/charts/erpc/templates/podmonitor-erpc-db.yaml b/helm/charts/erpc/templates/podmonitor-erpc-db.yaml new file mode 100644 index 000000000..fc0860b8a --- /dev/null +++ b/helm/charts/erpc/templates/podmonitor-erpc-db.yaml @@ -0,0 +1,17 @@ +{{- if ne (toString .Values.postgresql.enabled) "false" }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: erpc-db-monitor + labels: + app: erpc-db-monitor + cnpg-cluster: erpc-db +spec: + selector: + matchLabels: + cnpg.io/cluster: erpc-db + podMetricsEndpoints: + - port: metrics + interval: 10s + scrapeTimeout: 8s +{{- end }} diff --git a/helm/charts/erpc/templates/podmonitor.yaml b/helm/charts/erpc/templates/podmonitor.yaml new file mode 100644 index 000000000..627f8372d --- /dev/null +++ b/helm/charts/erpc/templates/podmonitor.yaml @@ -0,0 +1,61 @@ +{{- if .Values.monitoring.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "erpc.fullname" . }} + labels: + {{- include "erpc.labels" . | nindent 4 }} + {{- with .Values.monitoring.podMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.monitoring.podMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "erpc.selectorLabels" . | nindent 6 }} + podMetricsEndpoints: + - port: metrics + {{- with .Values.monitoring.podMonitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.monitoring.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.monitoring.podMonitor.path }} + path: {{ . }} + {{- end }} + {{- with .Values.monitoring.podMonitor.scheme }} + scheme: {{ . }} + {{- end }} + {{- with .Values.monitoring.podMonitor.honorLabels }} + honorLabels: {{ . }} + {{- end }} + {{- with .Values.monitoring.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.monitoring.podMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.monitoring.podMonitor.namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.monitoring.podMonitor.jobLabel }} + jobLabel: {{ . }} + {{- end }} + {{- with .Values.monitoring.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.monitoring.podMonitor.sampleLimit }} + sampleLimit: {{ . }} + {{- end }} + {{- with .Values.monitoring.podMonitor.targetLimit }} + targetLimit: {{ . }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/charts/erpc/templates/rbac-vault.yaml b/helm/charts/erpc/templates/rbac-vault.yaml new file mode 100644 index 000000000..55391c698 --- /dev/null +++ b/helm/charts/erpc/templates/rbac-vault.yaml @@ -0,0 +1,35 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "erpc.fullname" . }}-vault-secret + labels: + {{- include "erpc.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "-4" +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + - apiGroups: [""] + resources: ["secrets"] + resourceNames: + {{- include "erpc.vaultSecretResourceNames" . | nindent 6 }} + verbs: ["get", "patch", "update"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "erpc.fullname" . }}-vault-secret + labels: + {{- include "erpc.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "-4" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "erpc.fullname" . }}-vault-secret +subjects: + - kind: ServiceAccount + name: {{ include "erpc.vaultCreatorServiceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/helm/charts/erpc/templates/scheduled-backup.yaml b/helm/charts/erpc/templates/scheduled-backup.yaml new file mode 100644 index 000000000..d50daf0fc --- /dev/null +++ b/helm/charts/erpc/templates/scheduled-backup.yaml @@ -0,0 +1,20 @@ +{{- if ne (toString .Values.postgresql.enabled) "false" }} +{{- $snapshotClass := .Values.postgresql.backup.volumeSnapshotClassName | default "" }} +{{- $backupMethod := .Values.postgresql.backup.method | default (ternary "volumeSnapshot" "barmanObjectStore" (ne $snapshotClass "")) }} +apiVersion: postgresql.cnpg.io/v1 +kind: ScheduledBackup +metadata: + name: erpc-db-scheduled-backup +spec: + {{- if eq $backupMethod "volumeSnapshot" }} + method: volumeSnapshot + {{- else if eq $backupMethod "barmanObjectStore" }} + method: barmanObjectStore + {{- else }} + {{- fail (printf "postgresql.backup.method must be \"barmanObjectStore\" or \"volumeSnapshot\", got %q" $backupMethod) }} + {{- end }} + schedule: "0 0 0 * * *" + backupOwnerReference: self + cluster: + name: erpc-db +{{- end }} diff --git a/helm/charts/erpc/templates/service-headless.yaml b/helm/charts/erpc/templates/service-headless.yaml new file mode 100644 index 000000000..fbbef67ab --- /dev/null +++ b/helm/charts/erpc/templates/service-headless.yaml @@ -0,0 +1,23 @@ +{{- if .Values.headlessService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "erpc.fullname" . }}-headless + labels: + {{- include "erpc.labels" . | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: false + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + - port: {{ .Values.service.metricsPort }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "erpc.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/charts/erpc/templates/service-metrics.yaml b/helm/charts/erpc/templates/service-metrics.yaml new file mode 100644 index 000000000..0f99693ea --- /dev/null +++ b/helm/charts/erpc/templates/service-metrics.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "erpc.fullname" . }}-metrics + labels: + {{- include "erpc.labels" . | nindent 4 }} + app.kubernetes.io/component: metrics + {{- with .Values.metricsService.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.metricsService.type }} + ports: + - port: {{ .Values.service.metricsPort }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "erpc.selectorLabels" . | nindent 4 }} diff --git a/helm/charts/erpc/templates/service.yaml b/helm/charts/erpc/templates/service.yaml new file mode 100644 index 000000000..466c77f1c --- /dev/null +++ b/helm/charts/erpc/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "erpc.fullname" . }} + labels: + {{- include "erpc.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "erpc.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/helm/charts/erpc/templates/serviceaccount.yaml b/helm/charts/erpc/templates/serviceaccount.yaml new file mode 100644 index 000000000..0b5cf03c9 --- /dev/null +++ b/helm/charts/erpc/templates/serviceaccount.yaml @@ -0,0 +1,44 @@ +{{- $serviceAccountCreate := ne (toString .Values.serviceAccount.create) "false" -}} +{{- $vaultCreator := .Values.serviceAccount.vaultCreator | default dict -}} +{{- $hasCreateOverride := hasKey $vaultCreator "create" -}} +{{- $vaultCreatorName := $vaultCreator.name | default "" -}} +{{- $hasExplicitCreatorName := ne (toString $vaultCreatorName) "" -}} +{{- $createVaultCreator := and $serviceAccountCreate (not $hasExplicitCreatorName) -}} +{{- if $hasCreateOverride -}} +{{- $createVaultCreator = ne (toString $vaultCreator.create) "false" -}} +{{- end -}} +{{- $runtimeName := .Values.serviceAccount.runtimeName | default "" -}} +{{- $createRuntime := or $serviceAccountCreate (ne (toString $runtimeName) "") -}} +{{- $runtimeServiceAccountName := include "erpc.serviceAccountName" . -}} +{{- $vaultCreatorServiceAccountName := include "erpc.vaultCreatorServiceAccountName" . -}} +{{- if $createRuntime }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $runtimeServiceAccountName }} + labels: + {{- include "erpc.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "-4" + {{- with .Values.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: false +{{- end }} +{{- if and $createVaultCreator (or (not $createRuntime) (ne $vaultCreatorServiceAccountName $runtimeServiceAccountName)) }} +{{- if $createRuntime }} +--- +{{- end }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $vaultCreatorServiceAccountName }} + labels: + {{- include "erpc.labels" . | nindent 4 }} + annotations: + argocd.argoproj.io/sync-wave: "-4" + {{- with .Values.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: false +{{- end }} diff --git a/helm/charts/erpc/test-local.sh b/helm/charts/erpc/test-local.sh new file mode 100755 index 000000000..164658283 --- /dev/null +++ b/helm/charts/erpc/test-local.sh @@ -0,0 +1,273 @@ +#!/bin/bash +# eRPC Local Docker Compose Test Script (DynamoDB Configuration) +# Based on: https://docs.erpc.cloud/ + +set -e + +cd "$(dirname "$0")" + +# Source the .env file to get environment variables +if [ -f .env ]; then + source .env +fi + +echo "🚀 Testing eRPC with Docker Compose (DynamoDB configuration)..." + +# Check if docker-compose is available +if ! command -v docker-compose &> /dev/null || ! command -v docker &> /dev/null; then + echo "❌ Docker and docker-compose are required but not installed" + exit 1 +fi + +# Check if .env file exists +if [ ! -f .env ]; then + echo "⚠️ No .env file found. Creating from example..." + cp .env.example .env + echo "📝 Please edit .env file with your actual API keys:" + echo " - ALCHEMY_API_KEY" + echo " - DRPC_API_KEY" + echo " - TENDERLY_API_KEY" + echo " - TENDERLY_PROJECT" + echo " - LABELS_SERVICE_API_KEY (for authentication)" + echo " - ERPC_API_KEY (for Goldsky edge access)" + echo "" + echo "⚠️ Note: This configuration uses DynamoDB for caching and shared state." + echo " You'll need AWS credentials or IAM role configured for DynamoDB access." + echo "" + echo "Then run this script again." + exit 1 +fi + +# Start services (only eRPC, no Redis needed) +echo "🐳 Starting eRPC service..." +docker-compose up -d erpc + +# Wait for services to be ready +echo "⏳ Waiting for services to be ready..." +for i in {1..30}; do + # Check if eRPC container is running + if docker-compose ps | grep "erpc-erpc" | grep -q "Up"; then + # Test if port 4000 is accessible + if nc -z localhost 4000 2>/dev/null; then + # Test if health endpoint responds with OK + health_response=$(curl -s http://localhost:4000/healthcheck 2>/dev/null || echo "failed") + if echo "$health_response" | grep -q '"status":"OK"'; then + echo "✅ eRPC is ready and responding!" + echo " Providers: $(echo "$health_response" | jq -r '.details.totalProviders // "unknown"' 2>/dev/null)" + break + else + echo " eRPC port open but health check not ready... ($i/30)" + echo " Response: $(echo "$health_response" | head -c 100)..." + fi + else + echo " eRPC container up but port 4000 not ready... ($i/30)" + fi + else + echo " eRPC container not up yet... ($i/30)" + fi + + if [ $i -eq 30 ]; then + echo "❌ eRPC failed to start within timeout" + echo "🔍 Container status:" + docker-compose ps + echo "🔍 eRPC logs:" + docker-compose logs erpc + exit 1 + fi + sleep 2 +done + +# Cleanup function +cleanup() { + echo "🧹 Cleaning up..." + docker-compose down +} +trap cleanup EXIT + +# Test 1: Health check +echo "🏥 Testing health endpoint..." +health_response=$(curl -s http://localhost:4000/healthcheck 2>/dev/null || echo "failed") + +if echo "$health_response" | grep -q '"status":"OK"'; then + echo "✅ Health check passed" + echo " Status: $(echo "$health_response" | jq -r '.message' 2>/dev/null || echo "OK")" +else + echo "❌ Health check failed" + echo " Response: $health_response" + exit 1 +fi + +# Test 2: Ethereum RPC call +echo "🌐 Testing Ethereum RPC call..." +response=$(curl -s --location "http://localhost:4000/cache/evm/1?secret=${LABELS_SERVICE_API_KEY}" \ + --header 'Content-Type: application/json' \ + --data '{ + "method": "eth_blockNumber", + "params": [], + "id": 1, + "jsonrpc": "2.0" + }') + +if echo "$response" | grep -q '"result"'; then + block_number=$(echo "$response" | jq -r '.result' 2>/dev/null || echo "unknown") + echo "✅ Ethereum RPC call successful" + echo " Latest block: $block_number" +else + echo "❌ Ethereum RPC call failed" + echo " Response: $response" + exit 1 +fi + +# Test 3: Arbitrum RPC call (from eRPC quickstart docs) +echo "🔗 Testing Arbitrum RPC call..." +response=$(curl -s --location "http://localhost:4000/cache/evm/42161?secret=${LABELS_SERVICE_API_KEY}" \ + --header 'Content-Type: application/json' \ + --data '{ + "method": "eth_getBlockByNumber", + "params": ["latest", false], + "id": 9199, + "jsonrpc": "2.0" + }') + +if echo "$response" | grep -q '"result"'; then + echo "✅ Arbitrum RPC call successful" + block_hash=$(echo "$response" | jq -r '.result.hash // "unknown"' 2>/dev/null || echo "unknown") + echo " Block hash: $block_hash" +else + echo "❌ Arbitrum RPC call failed" + echo " Response: $response" + exit 1 +fi + +# Test 4: Base network RPC call +echo "🔵 Testing Base network RPC call..." +response=$(curl -s --location "http://localhost:4000/cache/evm/8453?secret=${LABELS_SERVICE_API_KEY}" \ + --header 'Content-Type: application/json' \ + --data '{ + "method": "eth_chainId", + "params": [], + "id": 1, + "jsonrpc": "2.0" + }') + +if echo "$response" | grep -q '"result"'; then + chain_id=$(echo "$response" | jq -r '.result' 2>/dev/null || echo "unknown") + expected_chain_id="0x2105" # 8453 in hex + echo "✅ Base network RPC call successful" + echo " Chain ID: $chain_id (expected: $expected_chain_id)" +else + echo "❌ Base network RPC call failed" + echo " Response: $response" + exit 1 +fi + +# Test 5: Authentication test (if LABELS_SERVICE_API_KEY is configured) +echo "🔐 Testing authentication..." + +if [ -n "${LABELS_SERVICE_API_KEY:-}" ] && [ "$LABELS_SERVICE_API_KEY" != "your-secure-auth-secret-here" ]; then + # Test without auth secret (should fail if auth is enabled) + auth_response_no_secret=$(curl -s --location 'http://localhost:4000/cache/evm/1' \ + --header 'Content-Type: application/json' \ + --data '{ + "method": "eth_blockNumber", + "params": [], + "id": 1, + "jsonrpc": "2.0" + }') + + if echo "$auth_response_no_secret" | grep -q "unauthorized\|auth"; then + echo "✅ Authentication is properly enforced (request without secret rejected)" + + # Test with proper auth secret in URL + auth_response_with_secret=$(curl -s --location "http://localhost:4000/cache/evm/1?secret=$LABELS_SERVICE_API_KEY" \ + --header 'Content-Type: application/json' \ + --data '{ + "method": "eth_blockNumber", + "params": [], + "id": 1, + "jsonrpc": "2.0" + }') + + if echo "$auth_response_with_secret" | grep -q '"result"'; then + echo "✅ Authentication with proper secret parameter works" + else + echo "⚠️ Authentication secret parameter didn't work as expected" + echo " Response: $auth_response_with_secret" + fi + else + echo "⚠️ Authentication might not be properly configured (no auth rejection detected)" + echo " Response: $auth_response_no_secret" + fi +else + echo "ℹ️ Authentication not configured or using default value" +fi + +# Test 6: Arbitrum eth_getLogs test (specific range and topics) +echo "📋 Testing Arbitrum eth_getLogs with specific range and topics..." +response=$(curl -s --location "http://localhost:4000/cache/evm/42161?secret=${LABELS_SERVICE_API_KEY}" \ + --header 'Content-Type: application/json' \ + --data '{ + "method": "eth_getLogs", + "params": [{ + "fromBlock": "0x1254048f", + "toBlock": "0x12558b2f", + "topics": [["0xe0c2db6b54586be6d7d49943139fccf0dd315ba63e55364a76c73cd8fdba724d","0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0"]], + "address": ["0x35a04d797164d54def24d6fc722c8f82f8ef0d45","0x1350994173f1cc83f8fc45a5af80a0acfc1b613c"] + }], + "id": 1, + "jsonrpc": "2.0" + }') + +if echo "$response" | grep -q '"result"'; then + log_count=$(echo "$response" | jq -r '.result | length' 2>/dev/null || echo "unknown") + echo "✅ Arbitrum eth_getLogs test successful" + echo " Logs found: $log_count" + echo " Block range: 0x1254048f (307,074,191) to 0x12558b2f (307,166,767)" + echo " Range size: ~92,576 blocks" +elif echo "$response" | grep -q '"error"'; then + error_message=$(echo "$response" | jq -r '.error.message // .error' 2>/dev/null || echo "unknown") + echo "⚠️ eth_getLogs returned error (expected for large ranges):" + echo " Error: $error_message" + # This is actually expected behavior for large block ranges - eRPC should auto-split +else + echo "❌ Arbitrum eth_getLogs test failed" + echo " Response: $response" + exit 1 +fi + +# Test 7: DynamoDB cache verification +echo "💾 DynamoDB cache configuration:" +echo "✅ DynamoDB caching enabled" +echo " Note: DynamoDB access requires AWS credentials or IAM role" +echo " Table pattern: {environment}-erpc-cache" + +echo "" +echo "🎉 All eRPC local tests passed!" +echo "" +echo "📋 Service URLs:" +echo " 🌐 RPC Proxy: http://localhost:4000/cache/evm/{chainId}" +echo " 📊 Metrics: http://localhost:4001/metrics" +echo " 💾 Cache: DynamoDB (configured via environment)" +echo "" +echo "🔗 Supported networks (all EVM chains via networkDefaults):" +echo " - Ethereum: http://localhost:4000/cache/evm/1" +echo " - Base: http://localhost:4000/cache/evm/8453" +echo " - Polygon: http://localhost:4000/cache/evm/137" +echo " - Optimism: http://localhost:4000/cache/evm/10" +echo " - Arbitrum: http://localhost:4000/cache/evm/42161" +echo " - BSC: http://localhost:4000/cache/evm/56" +echo " - + Any other EVM-compatible network" +echo "" +echo "🐛 Debug calls automatically route to Tenderly:" +echo " curl --location 'http://localhost:4000/cache/evm/1' \\" +echo " --header 'Content-Type: application/json' \\" +echo " --data '{\"method\":\"debug_traceTransaction\",\"params\":[\"0x...\"],\"id\":1,\"jsonrpc\":\"2.0\"}'" +echo "" +echo "📚 DynamoDB Setup Requirements:" +echo " 1. Create DynamoDB table: dev-erpc-cache (or your environment name)" +echo " 2. Configure AWS credentials via:" +echo " - AWS CLI: aws configure" +echo " - Environment: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY" +echo " - IAM Role: For EKS/container environments" +echo "" +echo "🛑 To stop services: docker-compose down" \ No newline at end of file diff --git a/helm/charts/erpc/values.yaml b/helm/charts/erpc/values.yaml new file mode 100644 index 000000000..d85b55494 --- /dev/null +++ b/helm/charts/erpc/values.yaml @@ -0,0 +1,260 @@ +# Default values for erpc +nameOverride: "" +fullnameOverride: "" + +replicaCount: 3 + +vault: + # Non-secret eRPC config template. Secret placeholders use __SECRET___; + # the API-key auth list can be generated from __SECRET_API_KEY_STRATEGIES__. + configPath: "secret/data/erpc/config" + # Dedicated secret-only entry shared by all eRPC config variants. + secretsPath: "secret/data/erpc/secrets" + jobImage: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/erpc-validator + tag: "0.0.77" + pullPolicy: IfNotPresent + jobResources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "200m" + memory: "256Mi" + validationImage: + enabled: false + repository: "" + tag: "" + pullPolicy: IfNotPresent + command: + - erpc + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" + +image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/erpc + tag: "0.0.77" + pullPolicy: IfNotPresent + +imagePullSecrets: [] + +serviceAccount: + # Deprecated/ignored: ServiceAccounts always disable token automount; pods opt in when needed. + automount: true + # The name of the service account to use. + # Legacy creator ServiceAccount name; runtime pods use runtimeName when set. + name: "" + runtimeName: "" + # Set create=false to reuse an existing Vault creator identity while still + # allowing the chart to create the runtime ServiceAccount. + vaultCreator: + # create: false + name: "" + annotations: {} + +# Legacy override hooks (kept for backward compatibility with environments +# that already set podSecurityContext / securityContext directly on this +# chart). Active only when securityHardening.enabled is false. +podSecurityContext: {} + +securityContext: {} + +# Pod Security Standards "restricted" profile, gated behind an enabled +# flag so it can be rolled out per-environment. Enable in dev to validate +# before flipping in prd. When disabled the legacy podSecurityContext / +# securityContext keys above are honored instead. +securityHardening: + enabled: false + podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + +config: + server: + httpPort: 4000 + metricsPort: 4001 + +service: + type: ClusterIP + port: 4000 + metricsPort: 4001 + annotations: {} + +metricsService: + type: ClusterIP + annotations: {} + +# Headless service for HAProxy DNS-based pod discovery +headlessService: + enabled: false + +autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 8 + targetCPUUtilizationPercentage: 100 + targetMemoryUtilizationPercentage: 100 + behavior: + scaleUp: + stabilizationWindowSeconds: 120 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + - type: Pods + value: 4 + periodSeconds: 15 + selectPolicy: Max + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 50 + periodSeconds: 30 + - type: Pods + value: 2 + periodSeconds: 30 + selectPolicy: Max + +resources: + requests: + cpu: "600m" + memory: "1600Mi" + limits: + cpu: "1000m" + memory: "3500Mi" + +lifecycle: + enabled: true + preStop: + exec: + command: + - /bin/sh + - -c + - sleep 15 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +extraEnvVars: [] + +# Startup probe configuration (HTTP /healthcheck on httpPort) +# Allows slow upstream initialization before liveness/readiness probes start +# Pod has up to initialDelaySeconds + (periodSeconds * failureThreshold) to start +startupProbe: + path: /healthcheck + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 6 + +# Liveness probe configuration (TCP socket on httpPort) +# Checks if HTTP server is running, otherwise eRPC itself is dead +# Only runs after startup probe succeeds +livenessProbe: + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +# Readiness probe configuration (HTTP /healthcheck endpoint) +# Pod stops receiving traffic if readiness fails +# Detects when service should be removed from load balancers +readinessProbe: + path: /healthcheck + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 2 + +monitoring: + podMonitor: + enabled: true + # Additional labels for the PodMonitor + additionalLabels: {} + # Annotations for the PodMonitor + annotations: {} + # Scrape interval + interval: 30s + # Scrape timeout + scrapeTimeout: 10s + # Metrics path + path: /metrics + # Scheme (http or https) + scheme: http + # Honor labels from the scraped metrics + honorLabels: false + # Metric relabeling configurations + metricRelabelings: [] + # Relabeling configurations + relabelings: [] + # Namespace selector + namespaceSelector: {} + # Job label + jobLabel: "" + # Pod target labels + podTargetLabels: [] + # Sample limit + sampleLimit: 0 + # Target limit + targetLimit: 0 + +postgresql: + storage: "500Gi" + storageClass: gp3 + volumeAttributesClassName: "" + s3SARoleArn: "arn:aws:iam::537124939463:role/prd-erpc-backup" + vault: + secretsPath: "secret/data/erpc/db" + backup: + # Empty = infer volumeSnapshot when volumeSnapshotClassName is set. + method: "" + volumeSnapshotClassName: "" + retentionPolicy: "7d" + destinationPath: "s3://prd-morpho-database-backup/erpc-backups" + sharedBuffers: "2GB" + effectiveCacheSize: "6GB" + maintenanceWorkMem: "512MB" + workMem: "16MB" + maxWorkerProcesses: "2" + maxParallelWorkers: "2" + maxParallelWorkersPerGather: "1" + maxParallelMaintenanceWorkers: "1" + autovacuumMaxWorkers: "2" + resources: + requests: + cpu: "2" + memory: "8Gi" + limits: + cpu: "2" + memory: "8Gi" + # Node affinity for PostgreSQL cluster + # Set to null to disable affinity (e.g., in dev) + affinity: + nodeSelector: + purpose: disruptable-database + nodegroup: disruptable-database-nodes + tolerations: + - key: "dedicated" + operator: "Equal" + value: "disruptable-database" + effect: "NoSchedule" + # Raw block injected into spec.affinity.additionalPodAntiAffinity. Use to + # enforce 1-pod-per-node packing across all CNPG clusters in PRD. + additionalPodAntiAffinity: {} diff --git a/helm/environments/prd/.gitkeep b/helm/environments/prd/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/helm/environments/prd/erpc/Chart.lock b/helm/environments/prd/erpc/Chart.lock new file mode 100644 index 000000000..ed8c8bf25 --- /dev/null +++ b/helm/environments/prd/erpc/Chart.lock @@ -0,0 +1,33 @@ +dependencies: +- name: erpc + repository: file://../../../charts/erpc + version: 0.1.35 +- name: erpc + repository: file://../../../charts/erpc + version: 0.1.35 +- name: erpc + repository: file://../../../charts/erpc + version: 0.1.35 +- name: erpc + repository: file://../../../charts/erpc + version: 0.1.35 +- name: erpc + repository: file://../../../charts/erpc + version: 0.1.35 +- name: redis-ha + repository: https://dandydeveloper.github.io/charts + version: 4.35.2 +- name: haproxy + repository: https://haproxytech.github.io/helm-charts + version: 1.26.1 +- name: haproxy + repository: https://haproxytech.github.io/helm-charts + version: 1.26.1 +- name: haproxy + repository: https://haproxytech.github.io/helm-charts + version: 1.26.1 +- name: prometheus-redis-exporter + repository: https://prometheus-community.github.io/helm-charts + version: 6.8.0 +digest: sha256:6f63ffe63edeae52172c50f6d1dd3ed8f12e964a7c8af088c49daeeb4f2a5cda +generated: "2026-06-16T17:08:14.874547+02:00" diff --git a/helm/environments/prd/erpc/Chart.yaml b/helm/environments/prd/erpc/Chart.yaml new file mode 100644 index 000000000..054078259 --- /dev/null +++ b/helm/environments/prd/erpc/Chart.yaml @@ -0,0 +1,50 @@ +apiVersion: v2 +name: erpc-prd +description: eRPC fault-tolerant EVM RPC proxy - Production Environment +type: application +version: 0.1.40 +appVersion: "latest" + +dependencies: + - name: erpc + repository: file://../../../charts/erpc + version: 0.1.35 + - name: erpc + alias: erpc-dev + condition: erpc-dev.enabled + repository: file://../../../charts/erpc + version: 0.1.35 + - name: erpc + alias: erpc-processing + condition: erpc-processing.enabled + repository: file://../../../charts/erpc + version: 0.1.35 + - name: erpc + alias: erpc-router + condition: erpc-router.enabled + repository: file://../../../charts/erpc + version: 0.1.35 + - name: erpc + alias: erpc-fallback + condition: erpc-fallback.enabled + repository: file://../../../charts/erpc + version: 0.1.35 + - name: redis-ha + version: "4.35.2" + repository: "https://dandydeveloper.github.io/charts" + - name: haproxy + alias: erpc-haproxy + repository: https://haproxytech.github.io/helm-charts + version: "1.26.1" + - name: haproxy + alias: erpc-processing-haproxy + repository: https://haproxytech.github.io/helm-charts + version: "1.26.1" + - name: haproxy + alias: erpc-router-haproxy + repository: https://haproxytech.github.io/helm-charts + version: "1.26.1" + - name: prometheus-redis-exporter + alias: redis-exporter + version: "6.8.0" + repository: "https://prometheus-community.github.io/helm-charts" diff --git a/helm/environments/prd/erpc/charts/erpc-0.1.35.tgz b/helm/environments/prd/erpc/charts/erpc-0.1.35.tgz new file mode 100644 index 0000000000000000000000000000000000000000..eb0368df394de9219f9f3b0961f55907c05c0986 GIT binary patch literal 16626 zcmaf)Q*$Liw}xZen%K5&+qRvFZQHi3i6*wW}^dP&AruCoCKsT z>>WTZd#_yayIP370v>2uH!Yd0M{g zYLF&Nn^U4iLj>s@Ib15CpwHswKn1>{Db)gO9Xxx^cQtD@K05$A8{cBScCEd=%hw)y zdV$vMTwD2bL`6>$RW$r8*im1_PaE!-_%B-bMNywoxK!+ygQJ$<5-RU(!eDNTw=6F> z(P2K3N(nC}VF$vUAy8Sx%EG|$N@63axV}OTQT9=RBvSxNE6fXsgo-df`7{?J+IGEV z&&G!B=VpDsZ`!v}5%Jh(EN;--`|h=)g9Fx6f9rDZGH}aFa}R{hABq8jxkTQsQT^x? zJB#JukdRx^3@*dgvrY?*yQQ=x8M-BNL!wmSU)II z6vq_j5QUAr37Jf?5;vmvaCi{;s}QQviptmOYdrF6d9Y{$L~NP`EtQZvX9AH3wyl_5 zG|*kW9eO?U8UCYaP#+J9ZJ^kgZ4Z5<33(W|h7l9TkInmOU9`aJNV9KrmouLNyF~v2iotz_R3*OP z(;SN%)t$4MtPXo|TD*qAH6&bAB7{W^Sn!yDv+k^>o7K3AQwcXrf{pNR9b_b_`6_P9 ztV3n%O-1HPmIr2~zaKnf`dLOGh8Qmc8@38R9!z5@W4x?-KGObXf@Y_fM$c+WB+u46 z7>p&!TU$3nMOxz+_d}CmIO`u|9BNM@iADZK_AB0IdK-X!G}_Z!y>K~92NxuVGro*oG@+f*Tf zUFKwzO((J>(u`!$CQxcJ3(|C_Zh!`>oAh@rxdz%$lX#Tqzzr-mIl*w^6c`l*Dp6!e z6t2FabWkun8U!3J9GDNyz6HF%S)1HXyk4J7<$NbqH5wIyD1U?CjDAeD1x=D#7HWiQ zW!-V#+kLgtawC4#3aDu2(qI5MQp%uV_#tcV(c(Qs zo2g_F4Cc7c?4z=y2byP7$r1a)2(W(aCXy@5>v$!cXK`k5ypSxEXlWOddRX5AVr-l= z8_qwa(}E&1<0P=-3TCW9&VLRpOZBTAX&ZgIz{Xc1AqW*A;rvJZkdTJ;nGawu`IcY_ zlR+AXwc~Nb_(_Tz-*wk^w1ZWG#s+ATeqNfgCxtC)3x{Rf-POSQYbe?VCo-9P;=4uR z(=m^bO)FF+8TF4lp(?&PJmwvRITr|0&y7qqm5wblR#u zOP62=e-6!ppFv8*e5pS;9>Rok3{pHbsc(W)V!j8T@wRm>r}1j>q_+xnzP()eL7L!C zgQ!X_U?X4|A5CMfp_x<|(RSaW6mwv)&O@;u?IF4+K&I_+_;D-@uq1t1JGZ z%zajdt2V*Xay>V^O&f-dg;@&cH2lp89&c}Gr5@|8>evh(kFENQ=S%xW#wSR*+k6JZ zU6=_NUpA%z4bvYieza&GA-iTwPiYT?HRdB@Nn!YYroAo#7tqMe3Fbgf(epz!fbCS< z=J{UyqS@J4@nTe%_{^i#ry=tSGE;!~jGs|5I3cU{uCj) zC2;{Uk_$Psc<3<*w6&!VeqS&^|7CcMS(H2qS1^}(=2+iNAp?NZ<+#~gC&9|Ciq6r) zgIHA~Sz}d7Zmn8;#4T=??5jmy1idv?%Q{x4oG5^7R>+m|J+ZFVL|k8v zznzRIIO%y|sxnkeq=HtXIo)s?B1wSr%-p5C!<@QwA){$dN=BE7bO4IAj?V7__k=P^ z#g626?6XhPD;)v_Dj|rV5+sVuwj-T9iydd@3A8L_DKE}j$YxS;hnwRG~O~WT+($>5Jqj-{Go^Elpk}M@H zTg9?=)+XOOhwEZ+u}nSSqrCj7 z!IW@xL)eC}l0BFudPb{SB`My8!!{F$l;%c9t=5;;3a}N>EUoZvCnBEnWlktV_7n;A zig^h$HphXV0Z7>i&hl0IQn3&C(q9ILYOjlB{+3#3TWbX;6_5n~jmomC!NSDD6i-H# z4f&E1J@Pv%PblrS+nm+tT8WG$)PQ&&hE43kCW=(BSad7%vnS2R#!aD9cycD}-3A(f zA=A%B?_9;Ndf3Fx*Y|y3#wcCYtFauT=p@FWC*`fbL2%}ek7TdCt-L7tqt_23QBphM zVeaHm>Yjrzz9~oXhRi8MX_6@=$KlH%rmQb=Nwlx*^0PW`uSz=o-6y-x)=7xt_J)GM zay-r_%dPI5#V7$)Oe6>#k>zxn#WFFOV3zy@Y+{*T0xuCPg)IvHX!P#UMpQx-g|aS&?;d}tE5z*~2$jbbguxJV(GE}2?T_a$IA!Tf z#vwzteOWeq$i(V?Y`B(8zhG=yJgcUCXYuUF;w#A&PqG>HR1(phuC~1ivotr%quoG{ zPwq2TM!tjBqaqy~7kGCXw=cWKM|Gj8bqaV^KQrb*kuY4Rn7=0(VAyp)&yJ4PC*a)N zGm-AqwWjBV1Y{>E zib&F^PF3|Mwk=~hE*WvEl33Wr=n;SbqlgVqrMqpluosWao7uRrG)O#|2k{4~OohKQ zy}wCaP|1pMf*+)=zmxeO^}CmjEo$HA^G0+GXxxX*8zvmp|Jay|iP7|TH^^a1uNIDa ztkbUs=0Xs>!XVV9yEyl`4B&EWX%X_~FEP7()J`rFarmPX$mjaC6iZ(ZMj{J~GDUrf zs^xUI42_$QxtgPJGMq6pY{|989&uqv%s#09$PIHWI};%iRL6LO5gJ$D6P26RuAmk- zyX+axmQ0h5iRRF9rM=5Od8ks2OA83shpBF)= zF=1OKx|b7HU@F&o_p3|O%-FSJMK=e4D>ytbmtBJks=g86Rcxa<=o~F$x`(nPcUxvx za-g20i=Gz*A40zO5~t|4-NyrL+yj1a!~K=7=ik-m+za{apRJ$G7yaH{Jy|vQzNzQ; z_-2YB?Q>al%u8Lk85E&~i>JYoo}2Zx>65eh*@_`Us~{MZC(ciE&tm5qJeNEmUrt%9 zy+FtvGHPqu7-?isWPca3(*6oV0_$HLZlII@IBh{ZNWaSJngdz_V}54ZOyG*x zmOooHNG}I604WBRsL&L?C|u%?`X*gg4MXP~7G5B`FUL%`RH+S;xPepK**K-H(u#m! zIFn^RmU<|FE^45JI%)YkwySBado5T^gC!nP>y0ckN+Gp6+yy7yJBAuDM@AT~S127& z`*b8^$z&Av7c5!Wl>+uEM;)%m22px=u{&Lj{P$!sKtm&6;~BbDaclKb{!)<%Hm*Xo zLe#zsu0eMc+#o!9TQkRjBC*fHbMRLW>TrYWbcisLGOsm(1$+xwYc*`YpykBy_?rc> zT-AxFuA!nQU8xSICWRt}bj*e)t~Cu4(>RMrpy+Z@0te68s>$7fvr+M-29o;nm`buk z>T}VRF`(SKyz1Veia{%RO?^Xu*_f?9Pi+(vb&~*ty#(~fydqgda=((84ZAe8YZr5(*wAgOL>D$1E5caJz5yk@m33uB&k za0;W5(*CAFF2giC%bQ9T{Fmr(e}+c|tYz1nLZq~>tJ9rjLc{<^J46N`tvijoCpcFl zD^|9(XsmoOBHZJzy_GZlN3|hG>!sD}W3|>h{la>IrX~bIo+-n)W{n%HQ3iB=B~yt! z208WF-A_b0me+ER=)Q36^sWdK%Yy;GS_V1L%RzS&V3N$4n z6fWG`I7xKnV_VGB^gnecvrX~To~7@6;v^Pt&*b6d*zFM%-T(=M>o!b^;dRtJbZ5dA zo9YJX8rAx^kXM?`t5vI~Qypbv`Z5 z)l*n>?(Xh47-gK1T`gR2vwAB%T9Vo{>kquVMq&m^nG#ltTBjw;3zJ%UimKBE768G-!w8IiV%>0FN8w)$33pRN8s;;I>&akKx@YHUqWx40EGEuF>gf96rel%W+k~k=WP^tQKoBI>=RCMSoUG>*{ zibPRpxO?+6PY$&cKR5{`is0Aak0q^gQY}HxE-4D9v=o zpY=^*)qU*yG?0^2ynoaMGU&;+4*ZbAs^|>!^=RhA_}ICY&3mvGN7y`v>eWtAV6Arb1b!!YK{Yy+RE2xV8|PZh zQzW~Yn!F$xm24g0vv{@+gu2WXNxgFQ4a!hvaA!Q-qbcWvx&4O>Eb5qiRix*78H!p~ z$c~m6aR(xKCWPWN=NRg0#@GP&D+A+*G3`ZrfAQ4|znN3nSyw#Fu>g|7lgqacd(AV1 zJ)>m32Yju;rh6FsefeiW+l5cXi5!<^Jzc>MI|9PM7bTNVt!AY)F)+xVmy(Ciq4>_w zSoy$qfGzeI@Ji9@!|}5}o=S&h98*VT1FK*JdqOn@PB9qrJbY1`;(;O>;(R5;pFtH9 zDR^pka;m3Tm&kJqxh4e$1z&C6X)2IZa(B|_mHu&C?k9;t$(D<%kZ^Ph&B+i(Z)qJg z9G&(w!eR6{ITlk z7{&VcuMBl|QF4&V6n5WidTHlee0pok6O$_t&I)_?dgk6|ZbW8SJU-&+9-LQR3LfD*Ox zmBgPjo#kQV?MfZc90pLzgZ|3i^1KcnPV{%hLzIqg*Lgb#l^#pW%K6;^Af+Pd+Kh9uzn1JXDJIBA&aZx-Oc3}!*G*eVy>HZFyIXzu zbN=AxfvOjeYVYu3q?jBGTqC78Tjf_1JUQ5MTGq+uiCFahI9oowS1|ReRm;BW*0&}k z4$f%_643!IMLgZQ;9@_1ch~hfPN;UE6Whlb?senW)AA5pEw!8qRz;CgBPosz*{qEd&Ws8JKDu{sy1PpjtK%rSBBF<}`QEKS3>@jal zR~X>1*?9Y%7H&iqS^IM)bId9K9&$9b?=dzp zJckEs@1XNGBhFaEzZfMMW;%KW&px1I|1dV587ag`MBZ7O+C0fZ0&Z8TOvNj-7Z&nV z{0Rbfd|@6*QgP0^rR6%lmAKN*a?gP?R}r;xGHv~jBc@=LiD}2HW?}w2sNta2UV^ch zMmoYJe#Sh6)865@xA9~}5ZCSy)oX5lu$HsZ4a-k(&=&-q`&uQrxaByn=qTP3jqWpd zg$PvGCVTEW+eduZbB@YwqI2tx*(KlE7~k63oB=0eH-~zVqh%Cq9XZMArfNyfpXeV{ zRP#zclYN2)qQ9gaOmP>vA7@F-Xofo#y=l0R`l(8iHdQIi9-@EoI_T6~CA_+t$NsCS z@+-n@wpuw*3eSFr8njdl+76tx#?&Dn_Ov=j+qG3|UC3Yk1O=Eh;+dy=hUAwx(jJ$* zZeEuIUubNSE>%>;DGk9n+jX>b{S8*U-*|r1m|zcx9#2uqpv)(C&hCd#7xC`ShW!Cw zn=*F*Fg~bN9k6GP-oR=>a%Ix|1PRdQ+^BhfNnW#fg5oq{0{h%R9bk9fY|% zdyQGePF}L2oZSjZ9#j`7@bLk|YEcYSoi*bqZbSXzZ>XMuLlSIxY#3*5EcCY*yw2OA z7!KR*B?};;{pZDCZN`W5{1;(YDHjo;Uw4O!f!?NY&o7VWp-2r(tW<)B?QX+Nh_r%n zm1e6|(0-``oGPu9Sa1m~4PRB^_4=E?c-IydGFgA0!5YdXXy4TCSd06l4OJvpu~2hK zb#qqBMq0`!yFECrWbs7(W?c`af#-bkfvp=VX`rXK1J#EiG+zKl?m1G>HF=-%*xx!C z@FVl;%_XU{9AfFH>_wSb^(j1(PPg&wJ`H3#?Bm<0Kv@G~ob2=BvbiAyEw1mUS}g&0 zW7C^ie2MVZhR8HpUUk@>fnEs;gkwPH3je`fT2!zgh%*WbqvlhZ-5&zDi8T^Y`CIp^ ziF$CEKJ^X_U+4R@@k|M`ffcdymWb91D{!if~7yPiupt8mEc%kpVY}+d7S+uZV4{$A8^N z@8-O-_ytaaT+VWm%!WuX9w$no8I7}CLX~l}RXqCRDl2#Syb67Rg z%0<;eGJ$1G_38b($j@;pZnNFpXFv4n z)hsk-ny_pP+t+;CvruzE`{!c&AZD(=uAv3h{tCeaVS&`0xSKFT# zJN+_taB+WmxOK1;8#?jBYEBn3FviR*B_fBwMO{O*YZez%i@>*hyQVdlhz6tgQ#db< z0{0uQRkc1zwn$CWD4Eqict!1M>#=65FN|s5&tTaNK#qNv{UrzhWYSRu1c$ywi_uVP zMN;vQchs<*`|h{C*26KHde#%=qGW0YLW<~2`TMQ}`n}e*u&533z>Y^xhM5X$yHKjr z@VJ@4Rd_PWoBXW;1GxMJ09dd))H8aI7Lx9H(w(v0Y+!DN(TT6t!wA(}=v`-=8PB z_~!Ayjx25Kx|R$(a85*fawL2N`jy$~UgGJsruE7jr*3xLfS9gKy&4BuL@HUKgabZy}G2mWK&Lushj;cL8JX$E^XRaR(0!7&71aB zlVKTwIa?DvguM86-uK7y#Oem=Fse!`Un(nAh1LC|D&Xba2D-CJtL~bsY)w+VSWo(x zCm@%jx+eCgqf3@bAvSc67JjtQnVDOL)Y%G-p~AUVw@ivY+a{m|s|EEgo%XT9)m9Be zVU&+6t41bk#AVb*`}o4Q-Y>FaSum4QqqR(yFpgP$oG}9p@X>g)YeZd-_E*vP27=Sq zQO;Onk3ChBFYK$`4w%I^-T`_8fN!~biA7viEc)E zQ}xjnb&uj@3b&wCuOPP%jjJ3XRTEcwwAihqhE?xZBYOFc`5*L>l~ddn>Pnqr8bHYaW;k~9enUi% zi`R7q2ut3kf9D}MoEd8l-do@l)WM}kcX6`WeO=l|g~7Nad{M5%7ML1#W!B8)^~We&;7NRf2eDtiu?`iOfR1k*yRQxKO3) zcJy(7b-Xj>@t=RKR7xK;$JcLvAZw@W!L&8hSg~Tu9NllgM&X(1QV);aH``RjkQhYR zM(qR)nb$3*=QMPkQdu3<$cfgjpt7;%P{{ZFi$S!PY0UB%3k;(Pxaj{JW;T6rc^E5S zy$OBajv;{=IDfBscvh-$1MO;C-8{eN>-o8Uey_z7n4}sWyyOb>SGaoy#0RXracF!& zx<{^k0t$H4+-G9TakcogM~DJWOqkaAzmHmB3{M8I&zucZpS!Fwwton@+Kngb_x@;y z&YER!TceMU4ocnx9U*VTn1kV%m6YIy0XQwN>{rXIRJ)A&0!CVi4BhgQCOC)6&afFLvUGx6;`7wwbs@h|gYI`DzU8evzqmab&QwFf7Hz zLJ9~fnc9eDf`(f$kzl2dT!@GnyyZLGdp0{6(@O$5uDU#P4>^IBYO(L%imHWmV}O}B z3NoZ?N>M{sxEvb9097+cp=9O`ghY!16LfCMOwV})>32jCi zzHxgL7TQ0w`r;a&zj27*p!Dl0(v>ZYuuP+I5>SS}2TUzcjVNYBXF#9k-iU{07&Qm< zVbvOMAv28Bik>;zCG3r>JmE00_)ef~8A|Hvc`qh%t2l-=wM;COc7*}yUKjt~d<;Hu z37V7Sh?Tr7+;`y5(CL7)*1z9{*UyS)-~HEf`M^MYrEj#tAg@+?%$uxt=&)98nKD6~ z13W|jBV-|nf(%}qT3qWyG@y4!-?lCwmX;io6mFza^S@l{%S##FM2rL8@0ACX``OvFm{r z9r2A3?ZV+tK`VIa^4DIi#ecj_BROw~v8K67 zZ@T@;#&9}DvqYAqY}_aKH+KP2%XLjXU1&hg_VE=L%3N(p0lU3x=;?(VG6`Of7a z42_n~xqPhwXq+g_7)-=(w`#Nyc^U2u z)xz`3o)k)?@ys}wnHF8W`nsJ*y8+V;@Mywzz3h-yA#@R6#0{A`iMM=?j!* znk6g8(1ylr?TIkw>2AjkT}|}k0ek-$9;o)E6tuUc5+&RCpBh23Z=7rTR4kr6umC7c zj_A`7i5xE;MdquYX?>XDl@2lqoFA)|L(qy5DJt83a!0hcCrufeFpLgPyf;MtNB z=*OVvXwAxUK9zu$uBO6wO%#3)SBJ6+dK1OKj?i_8O4V zZH?#=8M0NeG@dB*DaayXWng>Gc_2PR*1KNHGf8;}d<3aMP?-4ZaG7&TcVN)*A2k60*Vi}}a?zPK`3tH(vRf+xB4P37FDU18r z;icnctuy||qUf^1mZxS`4`qT++dp(3fKP2harV+O}ZPRDvlf37{c zGtnSYcC95*n(R+jFTk82sTyNB`eFU&@!XpgYziaBQrh3o_JH?2z6jLv1WMrv?9Jy5 z3vzsA7anp$j&_0O_(Y3o{*KjS7~>=Vhh0r4Ltsxx^0BOa2x8FFxnzy@09cT2kex>1 z?r}P+wpGGOxNIb`;xW14N!C?0Mj<;}Ce@7+!-cX9E*^-EVz3N_GAu~o>IKjSO3ZR? z?ye4+QlQA#Lwzm;wa>31M1#H1Sb5;KO9wSN=N3i6Pc-}&3jodx%C3I9(gV~2iT>ii z{L4)#bIEFGPRbtXyTjuVenxb}a_{Kk=fI-k$iqJ0Wwe?Iu7@$l$fyT*LfohE&-RVZVy^}lNw2(OXMeU^#nsy~8M4yzj zuhKrz8{IWoDj(%e|6Pu%g@pUjd0sl#!IBQMN)+8mL)%I0sG$a`gDK~&n7Q{PfTN46 zb9GbkSl>Bn-@S{>h;Ql3+kiQ_G2b0W4K(m`V=^|x!loYnI#B7X!Bc6UrU9t3+lkfS zlIv(p)FBO+SrT1+oUVovB1r8>Y6f!4}v?y=x4NR5Fmm7g5)M zExz?PRIXvQmXDaLIPaGPw#e%YfuN}D9=$xz;uyW8eGRyOz8E>te$7Krsj;sOGnOBJHFtF=agk%*_Fp zwkx|;bcn_9p?uHPQTzX9_>~^pO=;fR%~5K1&y~+^ak%C**uQ4j2bdDEt?{;Iy=@rV zJe;O*9M#pg-|#wxIaM@JbIC-}p0uqwjY23jEA)N(tF2S9%fAcK+4cUe_GydCGcGS8 zC(vt87u2{3GjjF)NO^3cm0d?%XfppgbDZ(}+4=Y$6JdPzzPng_ZfrpPX7Tdu-Xysq zq?U}XR|3b>uO7>tBN^ORKSS|LhK2d-Ri&jKBfo4gq#Jg<*sBQ-7#=XtJawl~ue!HgdF+Ekn&qSQJ?`Spb7~;y@%UqE*zRY1nK5hb{ z4N8`jTBY&L_Ul5?WssG>QES_@P8CY`yB!bldOxDV1YySm9rESD?U%e-$#*ufxKiW| zDdyH}i4rx^05BP28A3c0y1~D{yTq$=7e4+$Dy+-?N1iY4fj&Fa87>w$`WE1yS=CP` zw^K*pzlNv~75JfAJJ+AhacaTEokuDAAl=vjh~$nvPoFz&{MA{XyG>W8Z0MWMB%<~H zt!I$F*f@mvuLs%i zWZAGrhl6e>*=QA+@2!0M=4ZyD$R|ll>Ts^w?xErg*^eXX3Clvk$Kkw4#alXwSZJ%H zLJ9M(h;lu1XlNGlXyxOcI)7F8q5SG=K}g6nm7k7`=<&1x=|8O;`^Mks$|hbA{pi=( z2KM~f>DmJN(#<34J;l|n~7bue99WYycO_3OU(ct#<4YcGE+R~RA zRH+6ineI;`Q1wgb0qx|D{%CA$DdkW0plwiZuQy@ zKXafyEl6+4MbRfH^two z{;XsmckgUQ0@Y>yZYJM20w0=c{KK2MP{NS#qz-sl6zHlhOtm62I9Lbu{Cf1Z`}{Hw z*R<47t+yNEwf#5qe5|}1pRNfz78WU-N2koC?WHR4ColI^;ip&UuF|xf*acy3;oS|0r7V zwFCdGuKu&CJUl-fzkr&AX28NdW4I6W5%E$N%66mP+avf}7mNkmwV)_)F#pYHK_5zt z1>hacgcEA%jn(8v#T0E4$_T`;+EfWFBw7N_5*C$Z=Z1A%-9WOs7xw@*cTqB4KpF>+ z@(I{;dzUqPUnN4C14!}nMIzw#&}!5ter&75DKbh4kM0)1AgMWzA1E+fmxJG^ z|KL4fT9S3Z#12CKz@!F^{z~8SKy^#g3^$WG7Vc^%wv{S$`TJW2D8J-POP(F!Nua6J zq{Qu4n$@>j)m9LXX`_oN=2dDe#374UjuT7m=YuPoz@g6cd(S!!KRy$k0Bj1;R=AVL zd)!D%s%T>Kr>hogev*K_tPw*3YRv{|`LFGF3Cjz# z#8hg|z|c~3Kwd%Us5vu2mFG8o0TPSb>JR^yh-Kd=%ReN2%XW=?Xt1&Tm8bTcdUKT@ zIo@=1Co#MNE3(Ifx+-nkEOCBFB#6s&Q!^5`afHOg-F#lnl0kL9Q63WixbXKIvy~j)Qn>yfYQy?$S?{FGZ)p$t2%bHAji3ueG8Wk56 z`*2iCgrMQAL&)XDA0Zk>zZWcL>UH_9I*|4@{7l$jCT_12)?xvt25{m1iGgmF~W!!u=+Qc(5={jC=}=?zO{ zP@bHj;HBqQ;x8hPU0K$kII*cFN#mquoyZBHvknWl8ad$>(H%6K>Ag2a~GkKCgH*K3eFQNqaX0|&Dh!N7K^mRv1urt zpW*^?;gkyc3mv>(Jd~BFm2~C*pp;$CsEoNYj4lE>fz*+lJQWOxM^#f@t(c^7vSVB4 zpRbR%_P2jW2M@nD^!BuG$Mm8_9Tt|fFn&95;yqI&Y#!0ZmOoDpZE5*l_lsULg>s!F z{)2_LCl?*FN1ICZ-ZdAbBizsJXrwUGQ3g{mNEC}s<>23kJBzX3ddKvQ(@27uvNV)l zaCu>eAU!Se=m`$~WF}(yfrrCNzDkKfzSP3l9h^*ATLc;MJ@!h|#(U;+@5~`l}tXPs*Ap zZ~iVQ4?naeDJu(4Z7N5U!Buv^{5Ht^Bo{=>;##Svh9~l$^qk|eo^yQr&a7AccCzxlxc-dP^F{+Ut%0P& zIn7xD*3=WJ2<^~n1T3C3>d`Lh6;{D`g5VQ={z(I#{F~!RZEZCZped3+3fe@gst=km z0mqf!k8r`x!QYg&FMBSt`kH*KN8tXAQ)MloDgSJ#wwu^@G?hGb`^D< zVTtU`taQzjNBXhc5Gue`gn;x?fT!=vqy$a%Z-{5U@M%D19?~r(K5(h>MuY$=%PkWg zPvR9`p}FxZ>j}t&=abzC%>Coh(+Tu+>+#P7y3x%~Y2k~ZIc7z--oMPG@pFM|sqk`Z ztfdax)(Cfj9O{}u64F5DMDVtUzi=L{59P{Q^ahQ6OO_5cgJ5$V$!~XcoTIxhvU9)a z&gGPz)Z4z);@trFd82s~n}tR%V-j2ZEQl$@KAC`@$UxWxK%dr+ z*vW-8No}8maN&5EaQDb1Pv7>z;DfaTUa@K~7nfoPJVm{IEQ_6LpWc0RiQ~-WV4q%;)6pqV-&I7Eb@Kc)|649pWo0yj^547?x)_ z@Jd3e;6;$iCz>#gdMmWH##Hy3dx+@ab;RP>bUq6$fb$=h@H=dDfRTb3$?ms zGR^WvjOtw1bV(r~iIzM#GCRU}&x@uzL9&?ptMYKIFNS?cnLpr*Ra88uzg?dpW%T@_ zSJa<#`I=7^+4TA`?^KBUg>KXO?9abzI#*P)k|GPNSea}~3p;Gg%LxHF>!!IG>;?F= z?0g|GhJGjz!Ha|KV5k=nGr;5H?~UwxOEZa`_RYq$y8^`i2|U*fzmSO-Rie;$39ZA# zI6AWw0dbjR3Z#2YK8f}}burs3{wm7Xo^*JIBqKNc8|1NL_#v2LC63o#b~-uNMxB6h zoZUH3wWXdcW)!Jw8bg+4q9SUsKpyh}ca7?NH@{$WJf<-be-GE2p>HF!vQ27WwkVay z6K)E;p9jA7OJmO}mq~PFiloy^YItin36HzMO*KBZ|EMT+~bf7^T~0bjC$ z%cy%d-}5Wz^f(k6fXr5b_<45j?BwPxNYn#<_baqTet0?5->XIxmJkjssO{qW&u*oG7MNPT)}r0&L2B#c z!E!G$U}uYJoSxRe9hKG~R2GxcATH$IRvjvhUT(0KYgLalw!I$vOck@Lo%U&e`73Wi zV;XOmA4yzs27@A0Ik_cwN-2Hs?+}M;DuS3DDrFV)i~SHS zMN6DdvNmJNxFYiNdJ>E!nE}8J@9pC>%j$$>y=P>TD>N3QBK#`E-6ZS%jyimBd%>z7 zA_&MRcY99ORNJQt&7#GokXA{L0A_PAO03+}oZ3kC(f4lkktWzq*rVWINB01M{U^$( zu*mO5drM4A@PpmFXn!ITCF8&(QxDx6uC00oOs`D(%!gUn0HcM(DZ>V?sSh+h0>Z~V zLsir|O81vN!{|8Mo8s=(`Sk00LlQByBTMLt_{q(_x$=dJwOT*X z!+|v_j^yO;jN?0N&}1exWP6k)k&<8y-rw9RRI`m;ii(Qm1&mi%fD8Nboz-*4l78da z**Ml!W;$%8=;5O(k<0#XY9Qm7R-H;!ni(bwo(a|rNF>=5|K)Ja1OS zhb2=8DG3$!u{`4>lz%0-eBn)VQq(Pc4-UyKKdOK3X~_Mk%uus|oaw=$LeueNDuF?$ z%dk3?EcOr{Ix3vi{KY%vD7|d2x>mKTm0B=GJO(~XZr$%7@+DjpE~-*2UhRi#GlGZz zLnch~iZU0)VYJ~o&VhVvUDSiEbUT{$XtISLeGF||*0%gbYNw_!sbHmA}@w$DxHcp6(9?(t>?!S7ht+uw#yuL$|g~a)C z4R&=rb#I6eTx#1N7PijYIywk{^;{`-T)bT+?~49eJPCFDFnQUZ`@27PLV5lQ6Yp%b z?jW>I=z!eOqsP~I{F??TN}4F3qQboXC7)sC{GAWkb92ieu=e6`(Q<8<(3>wecPDf) zbaoM(-}dbfgM@rfE5PTqX=~>!ZnNN}h|P!jzFW^4KAx~m8Qz<$QDEEAli%&Rxh1OO zL=5_89TTX)bVd zg1mWS$ea<>;}z}xF1XV8>JIOjrYM~#e+3!NNo6`aP*bzIuJN^)*nWQ<;W>#*O73WM z0TO*C*JvEbC7|pr(j9~W%2XX(gbtO9^K4Awc*8&f^4WC*eKDojpsaXV@_HCu1;6#vy(I++>)WMJ+7dNI3=ULs@oKPBySoH z0~@Tf9*V5#I8N2OtEh5LZiBD5nFD+Z4Cy77&Y(;&AlZiK?{}4j{NCTC?5QaSpYg@Q zq)nhl{lWGguv!~^VMVqDjn4d z7`L#@&LfFM88ru8Qz zueOlxRoY;?p}-? zW&6;+BJ6$l$o;r|Zx07)_?xGZ#*wgTcUaWx;nVoTh|`J~=l&K~j6`aRF7T?ZpVM&D zuJYx)ec5N%p3{iWY1N2N9Lv73U-1(%+_H7U=gP0!sQ3z)X0t?t9JIhXjlxe>QDbo9&0BQi0|2%tK1+#y2 z*aLKF1)hpf*52%%$~fKI-|wHEs+|#s2kYupvI{5uQbZ!{uf8ewd&;2y!|6u0RBg41 ym$~^L|2H_J!0pXsrR}`F9KnC>RK~mK?zwyJp1bD?&;J(y0RR7U7or^iP5}V)!=26m literal 0 HcmV?d00001 diff --git a/helm/environments/prd/erpc/charts/haproxy-1.26.1.tgz b/helm/environments/prd/erpc/charts/haproxy-1.26.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..1fd3a048a4ec9f80c133b3aaecfdd515325368d2 GIT binary patch literal 32501 zcmV)*K#9K}iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKASoD@a&@TjObs9-!i1LZ6V3)>Sm;z)4WU2ugB?k*q)xYTsl z%rx86-E`PMP%$Ekn9hKRqGC7$Dj-TUf?~jMU_wwi11fT;h$n*Iuc|uE%m!fP&i}XL z&t;~&>Q&XNs+a3k9LkE+*c2#=Lq+p9!Gz#x_fIew4Cd$Nl7EB2VDi7AU?}Jb@{$+^ z_1Ru1%hpd?fI}*)@Uli86@v4TkN_-Rejq9-7BEQ!!}JJJM5H0r8qJ~7RMTkyd z0rHH&ApNo!^VXrJ21()6u$KX5dv$sQStIW?a6rf(D)0xHU@tF*q(NRp3aNk+kQc&$ ztCJL6RbR+1`4yXCZ%fzCYysZ?D6S&wuA*F6eqF=t%uz7(mPPKR2%+CzM40b3)l2 z^#7oIGC+|eHz|B9u7T`ecF>}~HHeLik|4!+q=MnYi@X`$3^1H$k*FdL=pu&{pv4g= zlpz~O=9?^FkUWs>4}#vffX{g4%RJYc0h%NoBw!N|B@O5*0$Q9`L6jE|K#eStHNcC2 zl@hYRLy<+Gf!E@s83Tyln*l}}AfyN;q5uL`lAC}Ob(RBABOS&c2T_EivxFuJw4fRo zURqRAUR}au`-7y)2vI<)3KTR^=M}_(NRvr8BLV^fYygr1U`#=X!_1EtL4(3;yco*@ zsua~4pn|*^fa6t7;Ul`{LYCP(UUe0aL;ywLE35{k)jlw|u)4H5%bNj4mDZeJF`@>H zDy*t1EUzgosRk8Qps1p}xU{CUqP!YZ3;~7ZqrtG!^5QIjcrA_;fEr~5_g_)~kI{%Y zjI(NlTphri7XMad#PU&|1p*XfI_BpwsU9gJFUEk3lmxHh$y5OpId29K_ylhV3#o(h zd%YPMpcqA=E@*(#6DLO}uZtux%ExpCfoi1G^DF{|EGy}vhF>ZuYBT6Mi}_E_z-v4(nj&(JL+k;Qr+NWk6$CXD2AZNHlkMVy zx+W!Xt!fvH8mSJ6Mq%9bih86dJcmdT+Epo;wEuI8X17?Kp;LXwC?P4$xoH4I)?QB)Vm z4#V=>`IPheMT{lrV2P}`HAZAnFZqjSO+m7x@|vVHxoVr@8v_NI z7ZD~78RkH!s7tmq#(Pl--a#cL6te0CK!dR`@cBUNp9~W#NF}px2+A;q*MW)vXY`+^ zuPL@{IoS(zmBp*ju~1S%DY#bn4{pb?}sAS7ZEUUj`me2)_$LrQ=Ed|ACy30x;=4NW3K~m}j z336&cmN+$#Vg6491Kj zD5yeF0W~JK?@+f`j6nqjl(YqLDNbVRkU|P80m$-+Kult+;*6osSPuZ*9bw=u8YbFO z4k^G&i3Ak6Fc{;bFMV0S$8h*>6pCzd(u^@{bw=l63^gFgE$V||T4HEus=t*^c0I!vmW>e8Vjs;jbr&uD8(9LfkW zLd4gLl$9k#bF@pbps}*?1lY@~Qc|=q$O#66);n2|G>MgjFsLc2w2@ZX9F@Q&>Lo!> zpfb#?Qrb1iAl*(@Glb7oLM`4R0LM2jx^$P%2*25qa3?^*AC_zYZD7#-7Z#>2F z6;e~`<)x{XpIgQogx{n_m^c?hl#s+~qybstKtkeBTZD)-I|rj4Q#+)n0h{0j+5;Nt zqKdSDw~oh4*f9bp^XX+j?}dlf%NkoRuY!p1Swut^5F*tGzm`OSGHC};f(6|11TR)0 z$Td|XRuVZPT3{I!bKGXI*uKciNbn+~;2tn4?SeO{OJNW+L_)H?fq*$ORAM^eHkc2~ zP*olFB`@8&Nj!(*(PGr5+>YgI7!mN0iHgDL#8g$C&tTpxz$P!QW{o!-2(S@J)R18m z!Xq|!3CBueeIVPPAD}c9_s<{K5<*6f7nsZ((@-B1#Y8Fn-U(P{ooLKA4v(TyQ(z$; z5*RaXzehm?0$Z5HZy`sd7%W?qu_8DK|xRX=8*iXCfF- z<5oNB5@w5x)gq&6P|&QsqCC;uA^m?r^ z{O@Hs~VGW}R9W^ip0CID4$WK*+m=guFbII^aMP74A z_4_$&BEI6;a!9HHFMg0Dr`kfK+N83Y;1nW#5nXMHNR8oyt_nVv^k|7dfe0@K(hGAG zAYdARh%`SdDafx%3E0R=qBtNwFE1yb6jc$cE4! z#duXyn%bLuV)d0FQ^Lr!DMm4x3Q#4zNI@JD@y=AKzE0nUfv5{YbIj0`VK6JC%UGcc zVIL}CnW8~bxP7RE8hI7}Dm7*9YQ!o?qYx}!WilvLL--4(8V5iQ42zY@h;)ndVhq0& z5vwr{H;!uBx&mI8{-B1*z~Yby1PQ3IaeISe-P|KHN=ezcO(yp(94Ytlg z)f@MCL2oE%c!o5-o=7e39umnlAt_X`RwN;lh%$m8HK>MJvI$Q?avQt|qIl+P=OBiZ zfEu+Gv%0ED2}@&Av#3ES3m}d4w`B?x6h#UWSrmxjiG&*B9K{$Y^MU%1KPM-UVf74m zjHwSXm|%Dn*ie)-6@+LR=C3fJUrEZ|Il*(NvPYM50%;1*stkrpmB%FgF9krqU_U1S ziUbCaGEMy?O-j}9AY)Bfk|o@-eK?mx6);=ynqN$j4;SXe7eQ@q!mvngrNT>uL<>;eH;&|5!hI zhBz-EfQvv}Drmks7jP0#1%zY}!rw*Yqc*8Rjw6c_3IK`qs;Sx}An_C#nD7#9szj_e zwwq&|P*o`@N5oE0S_ya&Ye|mEwn$M~QCwR}O)Nwe8KOuDlfOrKByd$IYFjMG6KXKQ z)KORcvc#3r9X3KmxW5>HsgxnH9@Ilc1yTGEVmU%VLv~RB5g^+eF-WZE6-mT9lc|RakGGU#MOla>Bi}DvQu0K{<3yKfd%|5XO;1S^>@-@L zsIaL@+BqNTv~wl(7w^_411Pm_S+BuN{>3rKBJHI#5H52Ue- z(-!8mn7Qp_mWeMRrjMW!B2~}bQqH*&WWZLG1ShuA)tvh zmJs`7gf#OyJvLm0asSQzLA9b}LCd{UlA~cTfx`@om~=Rab(4I*)~I>u6B_mu6Ql?f zh~gtiF`#Okq-&^2vmlC~rYlGVVNeETmw%>hhC}xX zBeuf=e~v%IB%r+S(0IJP4GW|{$KTJ(i>#n?NTuvqk#x)z5>TcKm^eoo8vKavdE%O> z50UXZ9%s|%*+vB@^XV!$9w$}6o0ex&>bM`f>SPCleyVJky*$6BrZN!nhwv*c z1lc%Z40~W0h!Vr%C+1taPe?2qD@dy2xrTXM4UrNt4GUn58E^aJ?*scsB(7;-|9EI1 zZLohJ9vW!Y$IG2?JjD6US^-*tKpIFEh+{JlKeRCuWM?GsS}~uLwYvPRy3MW5)D4i# z2&hCsl|IrjsG7tbm#m5ND%SK+ql{RMI=dqXKoLMifh2~8;|VUI!XuX&hWU!YNHL+# zGKClXgAUe0gqM6p995G|{Rc2ceJUQSs=%;F(fBBqPY{(6C`_Y(1{KCwHVi8nUZLiR zh^h*f^j|J<2w+(yM&z6WkpC$VB6W)rhXP*ww;BnE)g+r6k@jjJkL3%DiyBElieZq) ztHP{~X(*=~=6prcV{rgAO@RdE0R_p3=rRY;Uo07E3=}y=v-%M5+TR_sLG10mI@DzN zJ0%&H!K3n(Ow^$;rKFKpHPv`aq~S2|k$;V6C>lkq76!g@shW-qO{paMv*SUzRMLo8 zT|>P?{(R!I%JkVtD~uAIv&oGn)y5MmiCAmoNn}I2O=AW$+9jxhqQ*1k6R-Kjn%x>n zmINu*1XLLdLC!9L-oyz<+;O}Jl6`!c2Vx7woz;w{8QL`CXd2HdeOXQ371^&WGbz@k z>rg`&gvf-M6^7xNclTtHB+@;IMZgiF3N>Dh@>Ja}mdYi~oKI3nfDt5EyTFj|iIvH4 z=~x;Plx4rWB`W4QEGoPhQxI0to2|l&ss=?ieKm8>7nfHXYSaN6+Bi`qOU1ISaiUse zNQB0kFk7Vdb)x8#_#x`1(cZ#I@)tbaWIp>k0kC zd+TzB$`aR#7ZmGY#ElKA1{W5ALS2hX3O}jMmSWpX35ny2fw~qK$=^u zM>Z1UA8{%p(mIwaq$pX>)rQLFSk!IC4P8$Gy4olhV~r7Hn4pctmNo#BSf{F~#6hOZ zQN(;*)eim1fU*pggru}9P!+MPl#tsOCiQO_Ff+iY!m9Gp@}Xy*c_#1`8iT;%9N|Hh zeH4s{=%S_vicy4zq8(>UHiZ-g!+B9}^!p63z{G~%Y~CM18cd?VP!OaB#8oO1*6c_i zT^K03h?UZSp<(U|Jfn&OUUT3KyxP#9Ajc1&du@W(w(cc`5W3k)o;C5 z{WQQC!!;)CKrxwNxQSKM7;c)bFcNvWKG(0Ya;=g;3Cmb+2+z5}bMiqbKbRi`+4#E2qEr(?F@hkTcKJmiB*!MB|ROXB87x<2CYX! z6-k%vXJofjnt=}`DhmZjZ80Vw7RoTf3%tf#5u=qLCHMN6F z%ZqEPOR7ee7M0*P2K6=wvr*Xa15)~IoDDD|3uT>zIT#VE2cSV3;nXYz>4@HTxK7-W zv97RK65E<>1-H5bVx7UT?!u{MiSZjtinv*Ul4}+U)In5%DI2xv5S5g8#2v0)te>`h zsVTbXav;U6X35%vx+dZIMggz+l^0`aV(S47MkRoZFslS?v>z0Z!xc&lX+@PIMrgcX zh{2UeVUehjz#b#~iWVy)7-_jAw3LD}>P@$)HC-Kh)6;;^14l5zX5k_>A@MFhkc%Y}(Jngr$#v-hN8vD6?shNEhP zoY09M#fzNOV1+m3&q}Zc^Q&GLYYPrl!_Ws>Y)$@Iru3nNdhM|h%}mt z1m}}Yt7?*hYNQHuiTg%`CgC7wl!fn5ZL6m0^9XcPJQSV12Assz(X|)4SCMCR=P{fG^dk#n>KPUzTCL z8Q@DaX-L(W)Z)IWW3m(DGBa(<6t5~e2~+`tbuLC@`~O(x;E+qWNWfbuadszYAM7pZ z+#AWUJ(L0_c(Ks=jGQYo4ve;MP@QGtGrr(n(zR$9#7Zk&2Q{pxy|O^Ba*0EglA_`7 z!zIWKh5{5>r1bJq3Tk%Qs?;{Z+O`e@z?xubv=M>r>;`Dp1~UW+auEZx7XZQ0lB)~V zL=Qx^ca&~|kaRH=cwlBrC zwF)D&*R)(aB$qHaClm}BW2u4-j!Hmvji=>s>7=&O#7k3br9@1-8#$nO={6y+?Ge*P z&s_1+04T7bFd46gU}yzR8)nMX@Mz~59f!b2wV=x6$L>IACr~3~+UQ}@(Vu?lz&eAh z(?O@mGerao(MG4+iyVj&NAYQN$AB*y%{rGTjkU?i4jPeR8D@NcPIhi?j@Oic$|RB3 z3>_>3)JUR08|*U1SDq#6a>VPxCo0gb7@ZCyZk@6y|i9O zcPCUspjC-48gVq?2AL$l@hXeCPm_7pa(FU}aW;lTPL@cNLF(i<&c8d93^QIL?X!p= zMFI&(@tjjlhJQbcqr8vkPK1LL8}#*vvL z&3ME+v{*GM{fcQ-IM9V?s2~Ba@g=R|ZB#VRDwrIe)&Mkw{=oNy*_8}Si!uPS?x=iM z;|E!&F}Q8uoSZxYlBuB8Ih8PS2Mi2JqM?eHKz$C ztD~&Q8UIEz44MIOv^-@q4n&v-7tO%<|?Qa~}C8siwSM$0M3WiJmJ%9N^V3@J*f!{3B^T zDe~HHCw>A`r-oBrYcK|;SoxRyw3R|D^Aeak)#vw)mUKG}DU@XrVi5(<)m$Afa=17# zoB6$7x2BSSvW#UO1%uzyiXRh0Wf-r zg}~l%s18F-PMl!anV>hCm^yNwDM0%&P3UGl(lD0vk|3GoaGQwClDGM^@wz?gL;-*3NW%o>hE_UY?53o!pPGvw;GQ8;cT3z$3^@FIsAod_Y#+MCMVjsx5O zv&+C+L!;m$0@1t4=^0IO9OpMSHB^BpiHssiS{Bh6q7v4090P*JL7|jE zg0x8xNSt?RrD;KrO1em$54cKCB4m=r?X+!pLQQ$ZN~qJ zB@s(>1ALSHyoM6$nBX|SaV8T|PZ3hbgvOaaruw|zA*Mma5VC0OzD~|yk}TJpQ*p}z zGD*1z>UotX!|mftUF|A+fSIZZXoLS0(oBWLB&(Xk(6x{4nP-Ry7$g0mtDR=xwQzQ> zpqaCC|80q;6Kp$^g zQ$TwUH2?DfXMb*M?q@rPQQHzfd)p)H^MU7lZnuoj?UC=fm2A&_0$6q;>!+kq#;g;+ zCg%Z-`JnlNX+bP+(F_D`j_$K$-z2T990M{XUtH2klsp9s79tDsbI4adQLRU#d?WBN zzGTUc^p1E)NQV@fHQAA*nVOFsNsNh?MHe*QA&xkb;2H}w!i$Kb<5OE6pCPd@wz}aQ zlRE>nS*Z0ANKhTcQ1(r${LxheAlgXNq>J=TFq3z(;wkyUH@blixcEQG}E7 z8b&)FxC$bOg?JOIQUW>MOJoU<)EFw(fN3BkGA#JK3VKJV$-TI|8kZ$5D;cqwes5{i zHASYlqZ%?6dB$rrIkp+bLW~W>!$o`B>}=pN?iJ^l4D+Rh>7=PhdEb^rh$qQZ2T9+< z1KPgI?LJyqpg>&H4brWFeeImA@S+w4zOz*3EY+6;i#FQ!J=ctm@**~DbSxQYbN zB$-iTdC*eCgxVnLIg7l__ANE@>!f z$f;0hL^Q`lnkXQ1)tHvBbSN{=jJArjcLZ z-Ap&Tam#=aozT*doEHd`H%E!w#r*W}9!TPmy;BBkQ#XHRbFf6`2p#+1gz7WC_}T=Z z(c_!NvhyWCZv-Xm^{{ZmiYxg#9!&7sZ*!=-1cclRz$y<5)t&aF%l*;t+^UzKE&orn8^>p6cEYJ zMIxJNeM^TTP|qjAhiH)N$AjtV|YcCz21O24x|J0(BfO5NC2GxrG zpPe1bOVe>?F1w#3jRWx8r9D8?*vV9_-D z7PaGKxKNhWfW>v~E0PYr)UvN~3!dPxdDhu^*^)z$NxmD3G^vvW>eXh3cCUSI+?`<^ zKTpcrZXB>pbCOfFa>Rk;3~{tl+QFroIPF2^)83lw$^h(`769Dn+pkvonZZMjpd}M~ zRys1nLaRxmVM0jrdJ1R`nj<309i;6|+hWSai+&@V$O+dh6mTT?b}ct*M*=Kn-bU3NhMZ;6p(&{ub5W;hFzp48srX5M z52P85m3CyQ_S*EjH=SMAZ-Z&&Mi$4mY_Vht#*@jZ^*-*r7^vj|rj6jDB~#}tDP2?Y zZb_OKD`leHo}EQ8&7!c&yVpZXuN^519k{cvI!f(I^QKh`Xp%u!wWH?ROyw=3x})W0 zZjRZfcH00eHKf&=w&Z&$?55LPS>b3cZN-$iFj5OU4_c(XU=mm#)_HfV{n-aznwAQ< zHBfh7%y-VSABr@AoELNV!ph#`e8@D{=a&1OqIoy73F#w4yLKf`cFHX7jLPXoiS=w( zDZia<*;(F9vYa$>$H9|SNcP({3!yRHUUS2S$;CR6UVZHIQ< z`_bW2P66%mWJv_}N&Piuz-9DPeUHjpxHP~_a|TJN+c1~OBDx8ayUN}aZ&)g%-a(o!z!Y7?bs z(+*2%%XLMlyKKtsq0M$dQ~1OI_r!e#sJn3L^`L)jUz~S*Ceg*D+du!_2QyGg&TJ0y40z=aj`k~~x!NNXV4L~PubuRCG1Dj@^+Ef>xBpR5w;`k3O6 zOZHhMHD_I$)2g$pb#$~zGzTq<=gp%a|bm90d^-Gp9sa*R#Q*0` zKhT16%_{gO@UDSW`Nk%fZm*CmfZoYA?xdobPKnh8-{K!B&Y@;oPvdrJu?vp!t@dNT zor(^dKySR~Elz7E3^z?XneGyGh=cYk0?ba1>}XbTrv|X1U zwHY>^o2J7Zj?F)lV>8VTa~E`Ntquogn;e|&@0U!HzMHu=&nb&Hce7rbB=hF|aUEXK z4zFm3SG2<`+Tj)L@QQYLMgOmRMLYbEEsVOu582^|?C?YWpY=l?R8M4wFS7L}JA9EH zzR3S^Uu4JU;D1v6zi}D1+sRM*X)XVIc6LEZ{`Z1lhyV9rd^-HU9sb{T@c$AIUrSEl z%$9tA4jXNAK0v1#x5#m&w7;;j)L3$=|9q$dp96lNK0DR>?0W4sQ11)i2-09{&`B9r zx-H5iqmUPXucWG~qACo?lGas?UZUg%Ko(spNF&#n<09xvi7Pa3+EtSCT~SASGll;I zUU$h zeM!RGFg=0{xxhb^*Nm^3nJ)TH02qpZDhg{Nfy5@4*wK**7hTQqiomEhyhD0KtyURL-Uh|sE zv_h`Es1LS8n})fX6E0XGcr6&+(us4;LWJ(PwPu1!+ti9FiMMUT9@t)`jT>Oo$K#~7 znbctb$jwRuatze%vo=AHOBHSLjovSbo5J6mIkBnSMNA8p9F}z5q^lkB&TZm0ucO4^ z<>t?D5$u}RRJMT4p8J4K`(xjKX?CN=xKS>-D`^}G5y;jdkxLH!X^jxIsi$eYLX=&! z5NMRNlieS3i*CwO07N|zK?<#JGOP5CB};iv)2s3}^ucxiZR^do;@WFTRWX;dW|9z1 zNVCV&6mE9cGaZ8Y|K(4r{73asSgn`J~$aLml@24*UNf zX#Y>P>N`bx@=lEh>O)YFeV5njf-M)R0mS1((cDMv_#3)?y_|;G_nT9t%mWJwQP!xt$XL#MAqi%)>2);$ZqP@VHWuV8LXX!)3ozTy|KG z58#t3|H%@^@v5TBn6^Z8E_T3~z+1cjBR?L2oF z=lO#zJNuljsWcu%B?4LV{?j!19_S(2)JTQ@@oPr*Q3eWBCXcu+Qc)XZoFoCn-Nv6l z{Ft1QpCxHwwCwgyo8P7d2x&|&mXw&{PgfC(shD+4-A}BUNw*L~L68~{SE)#t0Fs<7 zVbF_{*gB*HjMLd!iY{t=0-3*i8D={fDhWxk;p2I26Zo|7+SFa6h67$0Gy|_q{c9t< zG?q*PSU%$Mhrgm^?d2wxZN!7^Gq7pAKkM0*X z3pEM_6@!hcLlsFk-2RG=CB`yc)v9!{ke(PM9)Cx+k_2oFgZ#X_oIJ~!o8ZMT2$_pF z?HwMe_jja*`bdhKRy7N#k{+9%ykCdq8QR^Hlka3)uBYkEaI_1hl|?oVNuNrK?FX`n zan63GV2qAXk>W%(L35$lL?rFuyr?%e_g0r>0VR;AL4iz|+8#5Ol-9+O|HSx>60+7* z%qzxEE18P%n`R417p9Yj?$Gr+LR4WCRX3@u<;_p`#?-;u6u{GAFYWlaK56tnT@dJM zJvjMagQ5JK|!H)YM56-7U|Lf5I+R^^oVph4dR~pl70Dl67gC4_45@Lz8PDOzU zsw7(1M*EZBG?Nd_owm-IH18mB=2ZHxNCG;j@n4~W{N($8bMx{${C@}K)8YT?p#SX@ z|F!>iJ11ixwV|$&a1zqG7f4y z+6zb2w1|$I)5w|8IXknCIGy0Y=xqTRus*|{2!f;H@%9*#lMOZDQ zcz0vUUxfq&RphsF%s00dz@yX5J;?(vgQz97ltzFY!u34wNli+LCfp2&0 z;y#Bera$#$HiH^9q=-fQh9A)&0y3dS5#0gi}rI4OPTjp4y7@Z42J$ z?0uLnc?eAK0&liv|4xSAe8mH4xXHCZT+h<440pBtz_z)*ODk3P)+nvqPn=3hbHN=c zbjHf%mHxc~8w!((*g0=Ri9;#DZ)Q#VrCq_141r{vNK?%wOxCpzDTqrKu#!SrWiyb^ z3}{Jv=R@lK4YHM~L3X5Rpc&NcEDjb3=OI`lpO6DD^t$`rXkzNvedR_epOv4(tVf!N zAuSnFGZWtYkF?7zdCEJ`w@>J6X-DkCF8v<}ziC$ggVYw#9%^8#_TQYGWczP^c0q^! zcQ8I3`d^3s*N*WYZbPDJ1x~i{CM$}jiL!0_Vw;VX2RnsZ(%{=M@0x+-q+EsW_Gp4? za;t_Ltu;~|66vXd77C=3Yps#AP_HAZ^iTb1mj5Rtk=G>UVDSH(f|U5r{DO}7&x7(w z<^S0o{J(?$?^ph(R?f5;6onUKepXVDgmw5tpg!ct4Q0@tGK2OslO?J-KyIPJ0P-g= zLW6Yoq`DE&s3~xGvIRQ54J=)qW*ST~6eE$7CHtD|RC&Oj7o(tk-D*k;7J{l%&&6AT zOXjVhX}!vS*zNrtGK&2%`M*DogajWGB?Yy68?EsFU|vr0`Tv|?NBr->`DB1fsA)(M zRl?TD0sswhB!Y;}3moAKkgbC;r24%XpawH+RhNlf7N~I~2p}d%5xO}EJ5-PWHNGB^ z48xA+n5AR@5yc2g0lj4fMfpa=Q7(UGrXN&@LKBcgQWJLxb0i?}BJz9v#nrXdnxr6a z1}Kse2}uMai>d*~E2`HY8 zI1q^OT3nA1smfR%8e-!C`|Jy2q>c_Kupy9us)m#RR8^#@0dk~8HR~}AC?ZHPiOCKH z8*|U`%VG@pjP`t)UT;Q*rEQ##gajbhr<2E}qNYfk&XNmnyoBzs_$7!)lBQ}3l(Cd* z)fObkfr#Od1W*N>mAx`xrD7mLHyAU_8V4P6I1q3h>&Y}dvp`od0mNeO7~DM@Lzr9o z)p({!96Sp5Gy4TNaPR)LjLSv-%Kwvd!*kjUtkT|{-_-FOzr%mQNyM^%Fo0je6$w%kI4>W2w9NfIgWIyI1qb+<7? zB`GGjdRHRGc+pXn6(pVW$4EI++0hk5$_6s+HOnhNL0GmY+?b$?As3p1n9g&UpL>T$ z3dS4OV}UL^o0dc%5$%$+31k6kQAtS2fBE%Ih79G}0omTxoQZVgUpz6X|yq66l`{ zV_%qX3ptK1R7zwNBuidzg~-xnNwU3>*VcB*0+8cKtjQz}S41kxMl)wQ)mpP~*4BzW56iD0vo^zoI ztCYC^4N%nZ?8u75B2(m=psO0FC@N(l5R)s#>U9=S>v)+OXH*Tzgh0??kO3Pwpu~fK zswgV;8&sQ44=%=1T3#cW73`<97C~rz4n|UKqKPr)t=2)h(;&@+Wmf<$Qc1s4T=eNv zTV7F9Qro9b7_`=nY|vXkG00Lzlu4~6O`--HO{0*C=L)Z7yMIK>#fTlxF!<|uAVUQv zkcN~Nx}K)LV5}Da3d1q7bTbMCRif)L-P8d)MP3x|ZDJ6L@|gOwEK<|0LbF#A%j3Ng zXBD#0nqfa?e;KCUF!Y7|*}488?W$vkVnh-2>LNc;M+8%3F+g3Dm>3d~0ySivWuo>1 zs(J7hge=2Kh>YdOi?RK}tJxeuaN4d$6)6EANk>AvMUO2vvwSIyO)XB4ZZhBA_y?(r z$%v5@5?ZfIL=7pj!m9`nnJE+Sq9y?dRD>6ihHMiGsFf9otM$8Aw7Gq(`J(e@tG~|I1{VsFzr~vHaz1Ymc7Uz{Thr7FsOC% zD|;0WNHm!k^>Y#yb_nq%#mE*(s+#V=d3qQv)znqI%@Sxp+2|3~B~|5xWhE^Ym!YaQ zND4QgY;JHOrWFSwRdv4X!I>J{oxy^SMlrbOs#oBBeceoVF3N#c46b zzXP0>>WZ?Gn!=%ucV;I`le&n3F;bK<-!lXVBkO{|7>RCD=x_3JSB*_noe!jXqzJGH zh?q4Q>6k!KSuuc0OvhxFSk9>_xOW1JiZCMS8W>$zHXN8prQC@4638Ybyc~*nA4DZx zemG_SYV=sFo6 zNJtzp`j4^m=-VPm8o`H^L>9@K8fcJ|I;_r+M}`Ql1|0fKW)_I(8sIeyUY1mq$MoJH zFk`F^9Z+kg$`r`NgKR*8(DD*AmRoiUL>RBAI$q=k*eFX`C8IEjBCyAk3geAY;-x`V zDp}WPbR}_80Hm@|Mu3wv)!aF@n;q1lCQ=G(9ndO3gQiuDco5PvE&H-$P$_Z1ab2=^ z%=wjtEuvO~YMmNTbdhl=SphpeJ|jyUW8S5TC3fOohZ@g9L1>~EHK@kzR6vSi?oMwb z;6=xs3|J61()5#|+Ugn^G32m#r<#=kb>xGZI8vy#)qpIiP$#;vx$zWd?*B-Wr5~`8 zg4GF$oVz2KrFhgNAnJlZXM?7Cc3c$Ord5;Erd4AP-G1Zj>;TdY`(2ug#s_GMj+!ab z^8L62`UO*0YZL?4Rij$^OWDqQ=lqgje0#0HHO+$ntjiGOemwjAs z$B)cr7N((bI%7$&o&r=UfdCTgc|{T_$rSiH1WJZglZhERtOPH81%U+d^F}aF5)Ef2 z=49yr`$07ofGHhiaT5>-yryAtgE(RejY|!vzD04@p6A?sq7IH1&cNUS!yTdLY05YU z;4>bMB^e0EI#LjhB{>osR0h*N#xxu;W{AIUfG_AL{{@1+_HKk#BvoZ5Ng{H-zQQ8J}wfrQS+sbYeN z8A_2L8^>eNBus;N(d+;g3lyyUC$Vm;Wzu!((Qd*WPnhaMy;O($fMozbu@#M{Qo?u; z5hS*btY@M}8J-NI&jhblAPhmKBnXH4e)+QAtTa4Jfby2oej0ASpmuvzdsI zl|&JQg4wxtCQH?V;{dDi34~#?0#E#)LDl_O5=F$OKBx8_rz@Dk^wSqLMlpJ4jUtH} z5;+h>wOHOJfFiuefp9;g>$HQl#`TdPViin%0?f7>P$UFGbT@{={eu01fQ=)zE~!j* ziX$0Joys(zh_j00CP{iK%1KMoXL6TAuhn(^jO9 z)9y8*abUqTb?4C^xw$z&Rk7-?fq*08VCsHsc6Z-f+xlcNAgY>pez*6VT_~b5)cX8e{>mD;d`n@d8t|I<04r-9@4=R!x$%a60 zvge3B7?FZldeq{$AzoB@4s%w(i@e5Ir>rtt--rgCv8p(>ABja8X(!M%PZDVYUE^l* z+%TBzJZPjPWcO<$>?eUZUQdKU8A?b>lh+BJ;v!ue&gsRIN}ne;g_g>S;@VPrT#E=bLlh}t;y218fwOK2w4PA3s2bFClF_BK(ne3Aslz52Wat9p z#Tda1(Nb8bV#4k&$HwX5Qp~WTgzn%)MnqF$LDhsNa;qIwh%)MMaEg)!Bw6DVQ1F8x zB+d~dowPzwVwR0d8e^>YD1w@K7Z;_d2T=i3F zFUpI!A~h_N4TUK-3l)3sgN8WI#%XI21UMw{^++*88QcrlSh&PA1k^ZGP$hk0$6{7W z94;m!MI$@T43hR*QN^?as^NElhB=bDoJ>Em**UeB*(Py8TM}Bjrd?)Ok_+Exs>p)o zq-hF@3aF9HGqJ&j0c80^ASN;NO*rHa75H;V0iyC^jckmMzVu}QAH(6lQ7E!azHy`& z-eSBMBfKd|Dli3La^-tT==Or%uH&n@Fk=8Guwq^nK#zqJ-<)H) z5h|0W98h&Ojv;HT9OxG$Tm^r{(lBlVQ#8_`RZ#`KN0b&*!x{cpK|n>sR2m8fgQl6p z5Z1CVXp;>TleE8v2OthrFru_LfMG~cT311Y0&z}HVP=+7awKF13S{L$g}5%PQwdN7 z_7LRe;;o70i!fzgmKOq-cVBT~UH0L)2E%&h zxcS052Lc@I$7;*f+Ws@+|31HMFEdMaK`Ki$Q#7K5W9=Vr} zlWMEv9t$Vc_C<8HDIzt76S^w+T;$p!a)Agh2GUbc3J@?2Kt!6Km+N!=yj-h`Hesm6 zgrSy5lLLaLc9=X!0NmG;kg=LHYlIwAAZM~vJJdxd&nZ7|dr*s6t1Uq-xN*@Akw|Xe zd==o3fV6$>h*2py1SKN_6r+e9i%~&7#qnr`b3j5UrKoMBrJfHB5l3-=GokEvWRGEt zCxFcFN)w~#qSaO63gaWXMtVC6VcJBU7lJ_K5c-mMI4WsFR{Yu+5^pB(B2vfo_WJ`x z73DQmrGrP*l$HG{P~9*<|Q$K&bW`3R54Gvt{gzxQ}N zN6F`pRz03Wj>rEWGCBMJ=kWkfX>nmq4c@9!G8WO&8)nOAPQZhBSCjW@s1 zsmIoCgNJsj$-VQ=o-;10tU2-gJKw&rruc%d$7fu1?8qB84L;_|pXR>y&oOW9I+D9% zw(sk&ejNCp$N#=Kd-vS@hVKV9?7AYj`rj46x8*x@`s<}@AMAh1g%y*(zkAfjwO`*U zpZVC&%Id2=%!vM$(Sd-*xrq_pZ`kDL8BC=XKuo z=f6Iu)2s*HU3&SjMNj!QFWq*_qbo=MGXDB`vnEev;)PxI9VI}#e)|yvwl?HU{`;HvbvyHcSLTX6ORL|$ z{NzQSKDFhh?yJVE{l}7z9^bh1=Rwly8+R@mxuk2)&8NP%VO-^uE%)hTt{XCK@mZ&B z|L_R*wml)V?E8V+;JH^#coFcf5z*3?wS4icwSyK;qq9`Z=D9sU66UG2UI5J4bFS~B+q4o^WHlB z`3Gjs=&|MMr(Q1TwdeIN-A{e=npet>J<c>D}z`ixmS z*iO^Nz5d%hGu~TzRo5xE<}JDRaK>}qtO0*MfWgSdQ-7ZK!%@SVjyS6CI`nkb$%p>1 z?fjcoM{Zz`n|Abk{o;yQeSiAq)HOd&zwfD)59pg#Zyi1R^bs4_qo?kA{441{C6`># z?fJ;?>F1u5sQUNnZ^E9N{xYj+`0*E?=_!ewb3sqEG2Fi>2R--UbsIKL`j@on@{@n- zdjD(TpK^MvI^pq?ep%G@;gwJ3#CE)YmB(}B$rIi>y5f#Ic2%!EgT6ze}k{x_74B>&}}pCnRV&%Utg?Ub~qD%;rBCU zXMOYj5kJpdacanS_w9!-Uh?2wr`$I3;nlA-ZhCm}gR#2$n#vPnW$Iaf`*(g9-}PUd zvvpbBw}WDQ{nm?axcLxI@UeH|pN#Cgt(#sDd3p9gaplR2?mhgN`(Kl$teWxO zsb}f)3tn6LjrTdFD%iQxlxf?4>@s~MJO7R~A1s*i`;neX_LHIauGn$cf;-DIZhGYS ziP554C+I^D?IUpyZrkGV+;=OB=78zP_I~r?SASS^Zb@+c^mjIVv@Y?((fs>Uw|=>6 z`i3p@9Cx z_1=fCH;g#c^Xlenza6>tlkGEq{r2J!YjQpEx~1#pmCUZbY3&E^p7!R3Wjh!3{^Fk- zl=Z(%xSr1}?EGQTuu*#t^{nb$bHeiAm(SjlyX>^?_uO>lPmle3l5(u)vg;pR@!g1L z2QkAIzI3$bstea%JL1FH*WVYt^v9DIeR^qkf61OR?^(Hf%Br8%?7lG0cy1wLe z(799plXqvPrw?DXI`88W^yt3z0 z$K~7_TQq47cj-6fD|Ytle8{NZSB=dZxtbjvEUVdb#PpMgJ`%0>-RmpJ*|u!=uY-J@ ze*X5xN6$WD{H!x36@2r<#hwd#7A@U4=cns0|MlBZ>!(+56hk>wez$x3@1pdE+&&7wo-u*X?H`bu2_{Ayea?lXU^U9@t`#~Y(2!IU*F}*$ikP3J1J`y?OOBcX&gCG!Bmb%^>@wSPSHa#BW-pfN_;uySEooRa5j0I^Jm<}?{rw#@a9v>aWzUJ<-te!B z=j>j77F#}V?$<-R9lGP~XYc5Cr6;5K-Ja8C&S^Y;Pw#I%o=+RA9H+pg?d(Y^YI>b2s@p7HU>W9xR-f0OgULw*vBUazeehEK-Amr?wQ2XXOXKe@>$D6D|)OXFK!#XejZp|Tg-OF%4jtxFG?YJ)2&gjuY_N%mW z(D#Ai4c%|}>BaurpWk@f#5eQSPkV9LJw6Y|eH}dR%@eDheY$IA^fAIvr zT5vs|d+j~H?Y#Wczz2hiKRxN3oXsEK{hIff;DUimC+)uT>c4I7bMNs#47;++?1F2$ zd7jLae|e+Jz`1WUOdGf~aO@KwRBwzIocrqE+5SHcJ8Q>P^EWM=85I^!+_3$ePM*%5 z^E{oOd}QYxcMiJp*HhoSVZkTAE?l@n8}jS5)wf@D&GRD%e0k&>cXt2J>tCI6?5bIP zH~O{O`iG`ZU-;lrhgW{E~jjYF6+&KxA$2% zU`F4K7q6H-bivaP9r^3?LysLf^{NRc&Mf@bwVuYCyLHwMed?OR?jIcH={B?Y=@Yta z-uv{*ZBPDk$Bl<={n-26kx#5q7nEJP_m^*^8$2HEt|j9Ssq8fL(CSsU{OGwXbmO6! zUw^-M_T0ILc;0S!>&i=3UAl5l!6^3FCU)>I^G^DByT|kD39GMtbmQFFSDaVd$@A&R zhCSV0h|E53&LPvzTRJFl%j=8JDf;-NBQH@`|1xyY?&YstKPZ>4x-n<!Kb#ek{G@+Gp>)Ug~nWr|8@9%ll{DTMwfj%^lKX(p3|BUO4u%{Ow;3`1P=>U)eNs z@sh)i{prk~mmKxToel9Xp6I{(y3@}dUbbw^@IH_8ku%qBe{cSSqso8%e(mZbdIV!H zKA-d4yH|8)U+?mlGhZI?;cMNSBA;w{Y3HI|4^}j7Tbo!r4Q5>aVa0-|FyY-jQ!d$b z$A(KU-qC-}lgr;c!guoLbrt&KQ(wGRoY~1Ee%ABCOFmn3>(M88PJH2tg1Lk9M&4Ih zam;JO_dMhAT-Nid<42v)wW52Oviw!wyX$Y-J79t=U%&8~(@Sn=j=QYtrbUIjJ!KaT z-}8G`UvcM}Bc=HtFS&N)%)dXm{9&E@?d!F3ei~3d^XA`=>3;K16SiMa|Kwrs9p;g{ z7oRodn&&+K{%p|l$0l_vIVo8F=&rBxvc&JyBX8S2;pZd%dE&qOPnR|oNh9h$nVea3 z|HW?%|M>N1C-fb4!@W;@wOiY~4P>hr{>*Nj>}>9-wkY}^s}ZMk^O%)g#` z++*V^FWh#{*w~fF?##X7sB6ZpYp6ZObHb^A?U#MYqL0`2Uof-#kNN%I&%d=+c=IQZ zXYa*}>xT7N-m7NAi=b}O#b@5x@9V`o?wa+)riqU{6;OVhJ7y`C0XNT|IkEnk=&ZRO zPxoiJcOL%gmO-cYIH|{}ho1L#r+++q=y_eX9r^n3kLQ2YbnI(?zyB4Fr~kTV{~0>v zwtpXAxm?e`xaf>duP=Cc%hl`o-#{doeaT>O)zRfW^3UkfzjMu`8+=!N{^~Cuy#3W} zQ^WAnn@a+@{rz?Z{{Hojue~PkM6L z(T93K=i|3m?788{%P#Od{L{wzsTptn^JLHSv;6P>J5W^f?oV%f_a1(G*O2G3Y4;aB z`t{Ezc*cKw|69F|n>VHFop8aU-}Wp#YVo9h3@tmo|1tG#~*s_%@ZEKvG33xYv28DV&f?T&uw`6H!(Ee zexidI>YyA}$-}CcnFJ))Fv~1_3y*qb|h#Ych!=>N&UoU&?r8BSi=>h3daL6|& z4LIYdZ6&94f8vH6Lu^JCH?iT*r$)4u=}1>{tx-wA72>B`Bq(YYiCdOV;>J3 zBu#nf(Wa+oefP@bf}@tcJ-=++V}0N07yHl8Q#==Cj=JI3AIhIv^ZD|@CwBFme06B{ znZMP~`;PTI`CQ6AggUUb6=6)!Jd_Ov#J zJ=;?X*!T07mfp#CU4QYRo=H`X-QE4ae(wDTkIBovyz`k?y?(#ur~EQ#@iSZM8s=_)XXW>Y&*&*^5{@{b$Abkkc0c#c`(sZVbIXpp zYvfPHPM$P6Z^z|>^Ip9A+V4*uw)wNrGZ&W4k9;0ockKG2+4psg@jw6diRhI3zv+I% z=7p(W%=f9ILr*I)gxV%?0>`LfshY8Cl+G@g3< z?1xUhv|sMM8_up;e@gayD_=VO#l<<9?~eRO-kleIaJO&P{GYcSx%{-gfj+$+{P?{w zN9OeCF=pW>LoQI>UbZmzl(Re!ezbCXVA6jk3|+rKhz|IPGXXMELv)Ay_I>6H8Qj_$#%pSg?Xm5e&${rv5 zzINno+cTxNrWCFmaoyDZH?F8FudMMmtzWtAo%PF}c;=|QRdqv$JlAyJ;+rPkzry1= z;*C4|^<33S=zf#$!m``*m!2}3-<+43|F5@aj5|Cr_>NuMwl-V`e|~-xo8^0KaNY&- zfZ93H&2NpH@zyoJ)Sdp_q2hliABWxeq&E4c&k~MYF!p@?QFU z$ZL=79`fi=-_ciIab)4(Q@{6~UwzzlXBB2`J7e{`-_Oh1cJle%9=thgruxa&&BtO< z=#3@!#45U;{lwXadHx@JZyl9awxxS-B)A24cX#&y!67)|#ogWAf(Me|4#C~sA!u-S zf+o1Te50zXYUsvDTb#l6I=lOGu_uYGqvG;GTIoBHdnQj|Qh4LG*kDNm~bxY-Y80%R7Zav$E^MhNB8kAeaHQp;+~3}31*p2_0v)$`K^ zkJthoCB7DEkl<7|7!6WN+Kzse9Yp(G!s4O}*(NGe6G(^hzU92oL~hFVR>PN$B2Vt$ zR8$$BoCJpT*0m75oI$gOmM#kgh;y5zNK9C*e+aIVworr z=`;?Td+_EaKwc38c-0(>gS%mH(=eOHjJI)gOFS5WFaU1esvpT_NE5|)Sg|=KBy4Le zDn_&VB}+dLG6L$bcRC%)4{90Z6KF39SYu}-o+>pGzI>0tDI)M{u+oN}av2a-_!?mg zwR~qTDDoAa8d9U*@%*u7IDU=b(<0lwb5(^PdaaMh@OLpJ&UXQ&|Fgs&$u$nLkIt3{jJXEweW)>PgijU*M@B93D1->Zmi;~L z@?Q;J>o`LdM%AW~N(o3MA5Qts;CvMXkPk-L#iyENh7oaxaRZyDoJxw~eHM~Un8yWx zj!qpry_>_Twra{YufrxYB7U!CvQi|gDMB#BPXt}Hmo6!k8^=Z3?gIJp$r+w6D<+bC z$aE_-B;mt5MlQEEz@WifY8ek+s~o}0@SP>1w8RL9JSJ9s(W&u3i0ULGKKmH^0kA_| z^U=QVzdm^+mpTPNNdl7)eul~E7;bOPrP;-+S1{+3wHK(j(2uCMBuPk@dfM_nnqQC( z!-pA$Qg79$K^BcbXNmy^r}J#^#+32mz_8=a>f1!m5={a^uu+Q7cpBYEyS8#<&(-%J zX1O#bk@CD8n+?0dwR2@Vf;@0T-qk=hwShee3rB-^B-9kQc^2hKp}1i0b~cg?HI~dW zmiP!W7AFOk=M3kQk2Cp8a~TYeuJj_2=!5GSV8_MkBk?kPv*;?{=T7%snO0D9 z>oXTTJ(Rd7XO}@v=rFR-hExz7G7b@t8ESne;j|uP?91VvPZO#Z{amMYqitMo-5MkT zjvLn`Gc3QcRQk#^v*{v2`c@Wd zq!DbP!8w7i5{)}mHMs%63=LGwNPCsqVLz9ptwHSdc-F+g6WvrsKYZnO++tsT?<3`s zzLPfF2LNBOZ9F!{2GiUTql_`wWR=jv5U|OxNUB4-usLczaGv+tjMd*!>+z&#!pwBW zP^m{fC(E?deCvXvc5mCIcX57+u@4Vti3`4VzcvA9dJs*oLwlSJflQ*a9-9kfOLi`a(&gN_XB%jK6q}&t%f8sCXAWBt z)T}-Zbm3zdhYE{B;D8tn)%EjGx@^htm}1IX(_CZjuMX%8E(V8{nKWha38)Z;Gd$jz z*|BI4<^hevf&E%pxFSY^cJIV!`UaWvD0sdth66yh%48QJYGvYPr1ML`yY2Mq%17Ui zkgn;_E=ti%+D%7_gIs0g^f1D+GiF&=2{SdMZI0QZ0=E&Lrv!!XqOF%ZAJYlRH^`g}jFehhuYb{$*F_>;@7!&x#NT37EGg8}*d%9&cc#fh z_B5=~wMpNxP`?3#LPKhI&}t$CD7mf2WFhaS8R^O|adb+9cS^pIg3*3O(b-U0u7GyV z9cm#%H!V7}HyM}&u0Fv64SaSY*)59`h#vtzl;9}#4C%ysD8LUS&{`|X5&qdr(W#mc zQi`i)Qc%f;qxbD-1&Bjh80E0*^Hik44PUw2wr5*I?&1U@2hj0M`Mo&L!0PL+U>>kA zrgPb*^;cGN)RhgfsH2mYWW&c*Q6S|8;wnV| z{_YcE=DTx*^7F$hIKYo@LUVcwir@O{smdMmR>HPGJl!@eiBB8{v;hdgoItwI|Se%JMhX{nVSan=n4E2qXx=;kV z2HaW%@5TBxq+F6FABMNe91jQli90|H-GWA5fS<#ZD&iBc;n{zUroH%;ax3S^LhaU3 znHU|#_=`xM0=V~QocUw|in0Z%Py3pIu$n65Z+C*HUw=6RjuV`Br5P?-9u^n1!xi`} z?>0XdG9{>f;DlvChvm@szb{&R79oj{gA&Vme6E_((VP;OuC%ff*6;;-NGgq z)dZK8L)&66gWR~NFnbvH+YSWit0Q`30EMKbk(|rh93SXlD~~*{FHB0sA9T&@`P zZi&R;i5+bvtz-u!Sw=ye<|SxLz;i(Y4YjumVyJ$;-JdWb@uGo+-#)b>lqgP5VTT!W znGNcZqTP#w?BEQv-XhZXL7l++9AhCMPcf;U&y%}xOmGQ# zre^V3K+igy(T5l#*0VNnxM|Lwa{$n>+1@|3$cnPgsQ{rO`@TELNWx<)kY}p1SCtmC z+gb}JaNS=-@~{Tg7rWb-U#5tKsBQO5u3BlXT5Luh=Bgr7$iD5aim30=rBxvdG3NYO zG`R|vuj=w~8m%>W!o}I9w(*WSfGzZ0AvvA1^5bz(n2s~CJ9Yh+$-I`jnBLeEFjyn`86AGU6i8KOXT6cR(k;W%AHR-NT+LpM}IY0=|# z;$RCWePuYV`zHw;8?N~tczhX40n+fBx8)GRA8=M@KiqThS9EPjkfD#^i?0{$pQlA{ zzR#B2IOwKxO|QzMQTdpcjE%fXOUH!)WXjYRI6s*5Z9dicEP|DN9@Ie~43E%aI-e9c`G zW)zR^)54ihU0Jg;Et_@3CyYyFX$jS*pn3ujfyo;<{PWvL68t`U7!o*?V_q@~3>=|L zTDj9Wq1?kSohvLBh{ZLa_>EUa_t**u;I?V+&Yr_BFGoefc38)T7;fv-I7RxmImuL6t`(y}%~K^!ZFHP!2kVpNd#kI7 zYp8RJ?$|1>iuEe&XM9K>FGz;96EZvK6&S6p<+Vb)l|_}r`f6wwO4XpXep#m$-K;eU z$zHTSRy1P(8j@Poew>m~e$Tj+$*ED_{7!)Z_k1C!7V2Kqw7gTC4Yht*;B4gyTIQnunis+eRP z$5k|Gj9e}WA%rS|_EWsv)=|e}O&W&)pBaV$$zXLJ8@-E=?~8bzueB@(Ag5qXl3rxk z=5=fYctKwfI>8dzD$LTUBRJDJ({MQfa!Xx#T z;e7R+?OtYEHYrt>RP77rQSDfDG?(ElFib14CSsqWEhbo;e4(iUztb2Ji%1mbR~$Qr zL(kqj?CLn8+0ANF&&i>dAQ>t}qL}8{H@*nXQR!dm@hf;SRG$t3goaKwAe5k>IM2Q< zy}jfHgrC$YzZe$R@ufN+$zeQnJii@6fM-nxhln<3(vw5-8K)n9J9sA{O+7fV&oUTi zXn7U@)qeEJvK+!sIpg|?3H^+KfRoI3ccr&@Xo`u$wWJLemSy(34Tp21x^zd3LiYYT z18&1x1z)4BT5rs9<7HYCJzwX^9O?%f_7kuKTX!EK93c;3`CCHhu(Pkb2Rlu&$yHxx z1P1exqr&_<9A|?)@3T$QudY~aE*&=HL&dKd`&bZqYw|}VuNaF-F(@!>ST4<50f3bS z1cyA~Yb6&EPDIme*KJ8>n}ChJ9$k0e=5uCax>~F(wp_ByTc1CSph5q{3vuXFEPM+K zP^VqoG>d=%HBD7MsH29Yr*q3FkR^m9pV1&)k|)9=#p_#cihrulx3lb7vY-=e9ciWE zsM|oMsF#4>?PtCOFhP358nZ{yWQQWu`xcE_7=T!sqt^SJu+pm$Mcpy3qhjG2h5oA0Ht$E1Q=U^4NQM6qoOReBqh;5?*fmcSo z;6vS?^vM|l8Os6TKv<aAb9DQI_08Q%#%{i+sB7U5wQJt6BGaP*}MpFIvk;TRTUOCu5@gjP}&XD$oOTu|`|D+f3u@-0w&s5$WPz9)(*E~U`-g->F zVXRapv%!d&sXfRnfs6Xwx^WUG%3+Y~T{Rb2rEBkP+P4jAr<0z6hFki=0=3AZHAGYW+cZwYaj|>V z1;CugW9h9j$Lupjs>)QY9zD}lRPD*8e5D7B2~@gG_uyU~vVj3G1w*wqqjqbVSYQBu zv)PV30okse3m)0Q#UpZUq+K3;?1~Vrs;K}tq?l4)nRVrlrCR#OYfFDxSWW~E|T?V zP;@+oy*_7FX&ko`u?CBvs0LG}JKjJ5>CHO!ksy+FbwUVVrXGY;NFZKkUl+b}_gA`E63=9tqI@(xEx zsKG$Jt(wzaJX@W4d+-^%27}Z z&DL37#64se$K_Q#dDg&y87H(w&C})e39J5C^@ET1ES4d;K6kYU5xd>3wy!%Y;Kw($ zjKGt@pD3IYLtddX8oB@{X(%YJhvuKSo1x^23jiLiSKT}eWOX!fIGrZvyovY0%2%J1 z!!la5r7o(+Uo1Dm(~Zz8_adrCi_$B&gN+#oA3jekAJ|~Yh6y095TAA8(h z*m-u_C4(*!A+t<+yRy&e_|nX0`#u7;oD2JQA>e6Iu|OiChPS9bj6@4Zb}nV3LYUeQ zH!PGoK6EYuT7BPXaQ`Vm|E%|l{)Q|tkKf00?XC;}up5qt&wP3ph4eo=8jNibfYS9F zrQoI$X$b>~*{ogVjP8Rnv+U9MS$bhSKikDXx}{r79yy>4xyv>VsG>pOs0k~RYsHn$ zzs4y{B^{9(t!Qce_?~j`UXIJl2Iso{}nFn&Fi`~ zU+APOM5lKCYfG>>mdAk}$dLEq*}E-5;v9RuV>x1%hJ%(lhFYN z;6^}P=Wnhx7Vsp;ZUK-AQo0R~rQV0$ZlK$?nW?0kmr7jRh@DYN3U=*-h&NdRyqVku z^M%j=_T~0cnQM~#S$S?@UeoJ6zQPFH`DHnL6@$;!dDXLczoh*4g0kF=R{C&&@KF%5&TLio zB>>Q4UX!fJksDZo0bXCjKjJ%BZUx)!_sg$*)$1onK6j{pJ+d?Kp)iBPEvAonD^~vI zcq)Y4g+G}=G$tj!dVh}}@RL7@h_@%M;E#Y5Ate#_OB@o~54Jt3R_OSac0O>IpU_%u zmgo?9jO4R``f=2cJpf8O{Q+Nr!aLE)VO}%@Hx2@IJ=^ltH9R>O{ivZB@fv@_(r}+r zdj*X<=l*cx@>!Bg`A!}MPpS+6j0ql2s%x|QXbWdly0&`sgH(t?JsEw=Lg-MX?cs~zygljHPMY3IK@@@>k|5ceEOeO|1>;zY*D2In z4bnkDN|7{=!nb)Iy5<8h4RcK|hUyot<+ldkaVYp!ZKvJ)b=r!~x+?B4DQGvb7DwkY zrgf5Ayi^9|igNIX5gm)IHa{nuDAeNH2JdK3Ke?kA$eQmko02P zE)04$sgOti2nDPqiCy&5p7(_6Jzmu#(=S@k1h5i|+;mxoehyWN&PYmY+R&F|BVkSu zzH$THKFzm$8Y6>+N&a-LUyB*$+1d%+K6qFAr{@z*VcW6Q^uO6PZx5usw74+EzwLV_uODE&kc6PFWpK znM@X;xSDXtoR*l8M6AVrMHXqiT z7a>OmkXMQc*=?TVwpy=OmjquJG+a2wn`k9zgj&!j(T6GZUJu1p`Xk6(1H?o@F$iD{ z&oiB0mxxNjTXQDt*F@E8tQ50F(`eS26i40p3(s|O#^lcT!I;&-fi;i`+u(<^ZVh6u z_FspQjPHgCqQg3CCR-aPnWw-T)vB{g_elacXc3EVa6m^fPaFK&)IbOCicemOn3|eRfVkN?t0An01#7ba1UeTq?3*#fUrY`(#5}hoq8S_@KA_@fh}~7 ziLZUXRm*0}G-u~@lF%sMcY~FM;H(UFnvd%(JlL-zC33Bh5a$@3QzG>8wCOY}teW@r zB*fUK8q}%2#~du`;KP$nyBTvBruAV7TQAR-i1VK$mek23(Pp#N#%-()<$FO_lGCvO zAo?Zx=BB{bA=YEbP)_N_{To$EZ9$wtqD24}&=C`qM1b8zb}x2%`jXLUUIkx9+c#M_ zF+w(RJ*MqZJyS6{?|iO!=){0()}#I>Z?Ll9D@ZNFw_(=)WXY<72xJbu)&Y1eJEuoP$= zekp$hs`ORUn2$ao@0>x5NI_@a46KJzcgErJUCZt9*|#NF%#Dy4D&|H zv9dbe#30IEgC@9MNYzUCP*n@+NYKVj#OBIL$4p|T0bsJug-_MFKyRNlrqJ2%RXSW? z%W%8U9J2RCx5uO>)gf<4yk)V4b5N>R4e6Q}zX~;dQqmE90TaZQfvG%7{|um(P-w&J zdDyPF;Rxc@nJFwkjicUc_a7`|l(FOsM8F~}H)w@s&yS`{d}pfloY7j*dRTV3d!m6- zHyzs&?KUN@>f=bpgZrbv)DhdApq$CQboATt8~=xO&37vCx$~fyxT@;I{zdpYv|uV1 z$)RZPffQ`$Zaf)jQMrjRQ-t-H9YoP5s%j-J9`f@Tc3m5k5F~b)7k4#xWW2hwij=;SOwG!bn~G{l z3N!fS5xOJzoOJ2i#qDTt!;V(l{mm9)RE>}`(PdynG3~1z8~|{b#?B)2Y9^A=Jk#)A z3wb}AhAA>_>^Xk_%AWaVZ;!m3VgF`k(3zE*epE!T22>L0X?603ep;knYvSCU+%$?i(@Uyoz51 zqM5w5`srQyNR*StcX0?UWaN8UN^$5lwpjf-L0ND)dz$RIxWUJyi^FV1IXKc7;l!l6 zS}_YoKBBw_Lvy`pBD%o6kuv`7Jn=XI$$rs~>pzAjd($5kLvG^5Vb+{Wi8n+E$vKDx|6 znORK%uP?XipT;hn)n`9WZ7f?THi!m9r1rYs9VfIGi5?h)e+j$kw5uDC5^l3|_GURb zhnCn}JTN;=*43^>i-Dw4k9JPZE!+`khO?`csRaj!A)f@F7(GlHK>`cT!FeX6Y|iyk zqF-%{@(v#sM;z^*HF~ssmV^>;CpUh$>0{9`Ij!SGfMlW9e@&-u4b@tROX(kec6X%S zAo0SDgA{dP+neAzmkYsw_gy8Ip8 z%L%0yC&8<#lPhr4w}%W;D>{gMv#|&fDfon?gi`%+hU9>a2v0`d)3>yz2h;ZAyb+FbINMKA+(~)!g;W912(sZs2EGie z`VPXr7kCbU-v&+bTi9Tol~HnvC|9%m^fgUk#0OsV3@ddWT#Frl+XLz#Dy)Uc5c0OE=Vbmp^EARU{ZjKvn0{p?^dxso z7P||NrJ>TH7p<^?oRG))L^MMagd+Ea(T5#=aWAGO7v^;FVy_q2K6%|Wid!>F9qT)&0fa^Ll@jX1+pgn94S-)xFDNLnoZSj$ww}S@s ze0SwrMKSg=lIm390awxuQtgLT5lzx6d1fod(OlsAy!nk|1DwL(&=*vXk(U zk;zr&b!GbcpzTCI$WT}c z8o@!vwVGsp&=#JU*SvAK-d^O|GT29(0F=j_8#x+gM^YYveAfx82Mf6=q4WfsXz3YK zyV2`<{2XLtl|*mZGiz^`!c5b9t@z1s7E&5n_mJnClPDt(@0^&8Hw@#19hK_iB`AAF z=t(?=kJ4jFQSG)G@g>+_+`6KQtR27znmU;5rbGEs0e)mdY7Yy1CmTCp#9!ZGKN)nj zK#z+fR}0$)3vv~c+jRsG*4IZu0wM2v#&>Bt$@Lq$Ph!bm6KvE9Ics55(Wb^E)kPn) zT5ObP`9Q4f*?SJb0Mw8G_vygSELi3|Md9P{Tt-vKGe2~|FAH|EizS8>Y@{`<(W!pE zZdnHb$bjhB=|;_aQ<3O&)-}szlz-^tA))iK6DP&v5Lq6(>K)8$ct&6Fl7aV&#Cj1U zw@Uq#*1UYf@4jilGxAA-+9{dl!2zu|l(NV>{wQ~Ac)F&!`*xw{=cGe#pM!Fho8JiC z_4Pf1y9!ypM&aF1fjiDWJb+xXYjVO)^(xNV8zH8!V5-R-Eli6G{!}u6+A#IX&4iFN z2-6cSng=OSCRlS|M8+hfQ!4-Eo5ouok*|?~kZ~WXYQcf`=)2?WFWl-SIx3;xWAR;t z!!%}3wxK%|SzMpx$6rS*3A};`S$d>6*NA_79jp*-egMyg_O_+DC)>4d(QO@DM#_sf zi~t zLsKvH(0*>uw-mWLSH1Q1*feiiczBMTGLX-@ESPLtnHRH2<=|hB1+>FHA@lS)F!)eV zk^lezNQujfm5J&F{HwpR{qL*4egAF#ch7GI1ARwxLxx|u5&zfj@n`-2nAv~Z|BI7@ zjpN7n-`_*O^=bLFp*Alu>;E`EgCF`o(;xD0X6xu=W9;--MSwr!pN)a`) z`Cas%_i6gIQxo^Et(v(1306)2Xx7C2k8Vx>c0(ISMO!O#LwDYPZwn`3?%?R8Zti6E z%e{ZZuK$GokpEx$$$T&VnK@W~$3GkM5B|T0e#`$akN)>2U;pZf{?I=~f6Tv~iN3YD zmHS^50RD`B7M9hCy~^e` zrdGzvPWpzHKSIR+K>Z>Ab`G`%#(&xJ|EK(Ov9tcR{^#Q0`oaJA(4W=+R^~3oHpY&Q zia)&)KSc!-eRC^k2V)g82V+MwTPq`8;-4!4=bycve>KYTQ?RfxcQV(v5;L~acULwx zw6!sEJH&Sl`I} zzu&ZeHWYQX`wu3?%KW`&^(RsJ*Zlv@81ygR0Q}SZpY^x-KPT6Z{lD*{U-7Q<7{OG_YduSe{cS8ZEo{RBjR6@Q)_*Dc zVQyr3R8em|NM&qo0PKDLQ`|W6=>F`#qTi6Kz#jI@kU;k4y<9z(u)EZfgaS7Axk{x# zV{2vTY$nT2d=xG{ItmbM7-75$@xg zj52{)_h5pU2xcgWzkA@*>-BnjuV1Twd%a%$-`-w-_q+b)qY{>+gE~y;rZ_ zdEGvKZgT1|==S7%@jATN+JSpS_ ztfat*GB}mO0|^$4gxs6Z1P@>$M8*f*?wE*4J`6&dbZJbcnDu&lT_tT77KV3)XV;WP zeBk*|%B1ImD(5ALqymu?GgSg7h{OXxEJP8(K}gexA@P65^0jQo14xlX0hv!_oqwZr z!U>)<6tTd;Q+?_C4#}}kNXF}u#tA)6Pm{n z3^9ZZBY`6i%wGc#EXOTS5S-BH42L-*Vs=7f63!YyWjs4f(J;o50mm_$k`V8QA(BNT~ph1)wxCpnz}%Y#We za!%5*f;-CNSbY~b$zmk1RW_c-u+jqus&mlyNP@;#_LAX@av~_34WLz*0AzU_YjO_Y zX!L=K6NWiXg+lt{6f;I5%oP?2Gzc_?>j~u;B0NHQEL0~Z7(!|3p;Y~{-vW&X(CK(y zNRtGm(EvVw@jQGZ7}{rJK7iXh&)a};Oou3jFz13M5ThZE`L=cjeJAf-Z8)!S@XNYrYhl+0+w*DB> zbVSB}FX#n3Ryrlav|c}mb1pDDI+47gObp;}{hhw~@xcM(@DQab70MK7a)2SB`5`%3 zD(HaFZid8U09|Wm8;hv6JF@XkV*wsABoXdN0q%Ik0N1onX@t*kj6*@0ej{ist zr2vgaBqd^|KWB_k#laYHuKOwFW@7^mrL;hTQZm9^a48NE&SE+P!~vy%7(+8HQ}1md z)q=#?hl*>`S1$psh?K)dIudP zM6yFt6Bw?)r>PvEL6XOUWHCNRY>d@-!diuGgP0HwT4HidJ^2}hB1dsNQz|9rIPw%y z`%%O&=LW?=x0}rLUwg0K{7p8k>88d2Du;$6K1w&~ZhLj9a%5@)77Nt;cUPL`zScyk z(izPoO;k$<43~PC?!S9?dI=*&6F}x`qPU5z2^gW6Yh6u7a4F$mDrHpxKsadOjew?r z%=iY9a)D7Km3tLB;b*FZ3;s8br7+?f!i7}ZMR}mi0=y=13}l>AhQaNl2d4djSBhCj z4?Cq*I+iHsafKkV(evZ8s(>_i{~yNC2`c}`uSlk9PB9}R>qmlme;)Hs&RZAAejJ;= zaETb8AFD^dfGUlrpsJ$$zaJt$#7qPsljbT!2bhVUNNmWc5!HX!h+2}ht@-1l z!w=^dzaIWmW6+L5OmHgvD?AIX@JwQ%Ui^w@kBZ0&w9tsS^iXdek0xh_r=Jc_OPaJ{ zB9$9v4oUQjMBV>w9-A`VIPn|qMNt7`0l9a5=ol;v1zl~n5D(v>uYB~GR0Qo4?$ z&`Ce}A<(|UrR zr3>|?u)i96^>LuQtKy4vno)?QAsm3gW3IjT+L)PU9XCKbYGt+dD%$^Q?1FNE)LcXg zkeNsD429U8NmvmoXCvB#tLex_S!V!=E~zC;L!HoT7%}{>9H%0lxiJ?5PTO5=9aDkX z6vYF0)#C*OskEi|R$U$tIL3*NOfA{uh%RaegIxad}FdF|ImwE@*GD`cuv z5E2(UvZRtGn4y%9Fq8g`q}s3f5VI5u%q;RB;i`aH`zA;1p|_r3?ZUrIw8_JZg+MhLmCIvdgDNXg2h0)Xskt z!bEpJt8{75r1DGMaO|oKXJ9M3F-a0bW&lkLX=)gTuTt5Y-f~@V8#3KB6roIMmqe0N zje`{9hzAzTdvt|?OS`0v6c-GUR4XKA*`Ox(Ib(9#VC1eIzHAJ4yMX9Jp3d1D9q@ zO))WJYJ8hd?%bwQAx{#-W~Mu827uBK!|g3dv|s2pg5v$%T?g*&Agc5mpc3m|4;0}F zS_^^S;?Rn5Br!SQeV{+YrJro=`y zAVHa$JxJ$?XjjL$U4BU87;ABmwp`ZKAwYmI8~-YdM`$5qm+Iiewal zp)C)pjnshT95dK7i*m3R{FkCF8qz5)s{Y0?YioN%Vho!&7zf+yEDUysF&%cN{b1)! z(36g%U}V_k$}#Y>>&TaIgRh*XTiP)*bjvv&<=Q7M9CB6>QV!QdOduko5msJ+MU`&_ zI-IE>skx`@stOG;H5<@ApY+Gw;FlCusj=c3-vbRP_`Xiza=NblK+~dZXmQ9%yN9yp zrnV+48Mz<~@fB^f!-&&qu}WYnRu=}EBM-6<-2#-V=mxyx=QUyuS-Iflu4`<-q4xG5 zqmf4hWpiRI}&|3K4IN`+ZjD_-kC((m

{OyZPsmwwV!x#(j{W6wzds|HOgAc{qyF2hHNF6P0 z#M{noz%v{M5-R`P`x2xEiMx)S@N&497rRx%_UNRBt89ti3u7X1qLAeRiJS*WuO(si z&iuIRC!5iMeME=V{!F(ID9J_R0>&p&Cd0yfu3gcI#BsD8@nj3ZAkU1IWS2>$i`IQ6pR z)Y;+br=x=gW-0iqu6}!T( z06C+h^NDaGEE)UKgARH@fAB-^ho0HSqr+#;wma#s*-lIJ{1Ii>h(++iQUzr7CPt`e zdz}$!B>()QHTnIXRqX%81jiX>JP_HvwgN7(|F3uVUe)dYo89OAe^2t<-gdu-DM5wVeggtlO$*PXhd$HxFNy*qg5!v*8FNe$B6P$?O<^~OGewk-iD`l6N+Mnnsd8UB9S-Pu0ib5R9PC4Q9)n(v7oUn(^_C9 zCP>Rf?jFD7~pz{~*|AlvI&~&@&NM76D6^V3M zEr*niI3cdtZDdE6Qt`#LSgz`IT07{{U;k~k*}?C2eH}V~?LgNg;-G3S^y0%?_= zQ!H|(BgQDp7|j?_`l{F{sT_7TL}7!xsMlrDD)A0bhK3|2f?yu(pB$M&Jop81TQwIG z%E-T!3l&`bpn^}+eg|AJTUNGx1TL;Q!~*pf4>luyG)ZV`cS5cqUb=>EVuIr&;FGQz zNCU~p6=cPXxJt*jc%jM&7@ls{b8QgV&yJ%F@#`<&b=C z4SdB8|4_vF9!_@$R!!ZT^d|%90JI zS{y-HP_fppxi+ z3)hcr6^%Z`~~{%dnX1JZ}@Fh2r#3VyK0oL9J6-w2G}R@}EixM9iSSc{FLR1C9omKO_~HoicU$S57ciKP-o7&pOQmt)GLOhpeV&4fR5M?ruTVIt%QR z)|F@j&UzbhsCp+_p9ao~yas#CHWiB>5 z6$-2JbDK9?NV~Z3a++4QV=kp(OKTNcE-zjpx2k8*l3QXK(H)Yl;nFR3%b>}HTHYEo zI8AP?2f9h~lzxD$7W=Cgv*EY1!_zeoYXZ2=xT-~`-K8eI9e+iK?BMk1rEBfda%I2G-57dUVG0liaw_CtVobF*MSPC zm`rhsIX_`^h^rDb5hD8;i@NeMSw`w_)#Q57JbVR5U|u+$)Kh@snCvg-=O<^YOo%15 z$us>PK))jY-no=;+jM1++6u1_+ zEhN)@`3Htgnkd^So-5(gByEji3yq??N*|g%SS!@uVvOeAENH{(k-5;)lNA{8Eyjc% z#*HT!6m5N3nIGR&5bD0{XvOMrxzW~><;?gdgI*8g#(#crwDx6Xeyln;YOiWegPS9> ztX0XJ+2ER1YRzwnXlSysfNHaDmJJqpINiqX0oh6LSps*ney$GxkKWGo{H*z`vj0z~ zLoM>SuDuCxS^ocizn1@dXXkbA_4EF}r+98}Yg>z|>3HRs_I+O|fGmbhs|B{;g$Y7F z5BzWXS!Mh?*(!9hiZwOBvh)AF-P-*hJFj|g_MXT8Q#_8>SvXxo6b5BBe%M@cIz-Ea zJ*3f{%l6+4d4OjHG`W&&u;k57@3k7k1M$q zWe-p6heIBsxR%kH;eX|X;piQsnJb-|D?aQWAA7ZEuMaPaVC_r$1}B-Ay(6r6QN;C% zH+q&){EX;zdW~4Le{v+>O(+*1u(+n|s>DvdJUTh3zBoC0=ipyt$x(1w9^(PLJe6T6pHoWJ#EQ?jz==in*}MP$`B};S z7iFaItbYb*iT&^Qck1?k=hfb`|NkV<+=&P`@vXZY;@a;rWp$v_>#Fp2#mS*+Lf}U; zr5JdXj1;UV9Rv5(s%5pI{kM^)$h?ts4!{41(G|E(5Pq{i^j0*!VCnp zGOg3Cr5P41pyjD0bjIw4_BGl(!d$mh&>bK#*V5}AcH9Vce~0A)i~n+lAkXn(!+x|=k& zL-V0)B<;jmyh5@@Qz6 zle5?vNo~NL%UdmI_dZu|IWt@c3#O&eaBFLUFADXx7?ZZS*uGx7UD>84{ZbUD+^}pv z7hfy4EITw=vsv9pOQXdN)h$gn9py7=%p5q|@WQF7lA5(%>b!!;wAWps_m%F+f#bYz z)9to(xzdw{^7APHLYjt1z-G{RF?iYCm~gSRLZ_X|m0-%|f3BgeQug;+*OR?!N(z5* z9K=nz+is5P3a6^Mme*O37GecGnbi;RFRQeAb5(uUphuV1Te+<{&9p#ewMNw1fwgMI z#aOMbnjfN>7voTME*x)UY5CO|-R=r3@1pAoEFXRVU+J(^Z1;))7k=c~d`ext2Kd`iZR?I-yD|XV@g*wo@wZ{N&;Nr7juU zjAJ^~gOQ};ZKZ1knPPp5Yxy3fk+%UU9_uugn;F9+a)Tp1q5fZ60enp38PHVKlqf-l z8N?*Tffu|xyEqe+VQ&M>6%U^d&LAR;d%>89uKKS=-wTHSW?l7P`(iTg%KzA(e42Jk zKtmK><(azsh1o~KlWzQSloso;X4i~};23xFk*6>iGjg7Pj3ps9pb-pQpQ z)$d2%f>2+JXm9qPVEFtCj+3%sbKT39)pVfmJYj+7ZEU~^!|JpNCjxWN^W9s(`a!?< zSFON)!~((o$q~2Lo_jvXX5iId6_}r~fYUND`B6Heo_Be98Pb%~7^^#HJ9$ICfd=v$@b5A~l5S*p9G zm~XR~M>NdU@e{RQ=0bJyDCGjhF{qsC^W7un^XJpUvQT~Ha9B2w_u24dM#1YrCl7B_SlqLq0_&mWO29iw7KKDCu*E9oX{!eAPY$eb!050 zNk&tg3J!>K8WJR|(JU_sQ@G(+RgiDZ4Rb-I>jfT5J71JsrTf{KA>}Bvbzc?0b*_It z_+|g}{0sQLY0G7-TX8wIy4-SHH{hJYZqN&Qp7#@@iT>N%nO|r)!d+G2aDrh88>bPD z%-TYU+M+wz=~TM^__}*d*%e90zGR+H^>(mssJswg1MF@Ea6Z8dkE9$V{I-){wX3DNa5U}OfthGY z^YY_ND_Y5@%Y{n~muhryEQFfTOA6X`1(MFyHJ>iG%SJVVHcF~4Hy&aWS&1~gXDEri zDkQx)qD(=NSFws%@wyW2(MdH?hKqTnn%?@5v)H6l8+5zLOp|r6_v+2x)C5v58;vM! zFYFsrSg$lhOrP&;2+-MSc|Eo}%ZhMvHLg3$Y)G^c;51EZ?wg_bD(D4$&yx~ggb)Q9 z7uQagLo$(OnQ%zyHC#^n!A)>;8R*$ag1%B#T&Zbh|H|`7$C&F2IrUwH>I&+q@#Kx8 zqGThb5Yu#wnVe9_@iHq#Pw80KtiYxcC&?lezoer!N#Xw+!~_Y8w}j+)R5~NgOQ}|P zMpI7I!s{%6f71MA9K$s#u1SQ?>Q2!J@yU=P7Omwob%!w>b`vD2FLjb&WY`@uG(su5 zC`ajqu2r-VeC5)Frt~_16HQx=wf^~BZ2*2I;+K2~85-lYFx$wj>F`I4CVub67u4_H zXg!`t#TH?KNX)e-c3{{*(y`|$GjX79k|Y1N+FDs4_)#lG(#K3^dbQ1C@Q&*y9Ma-4J{mrs}%t!f?Oj z@Oi3ZL{f!(*$leQ&#i5cqMfJ3?fb4AmUr>bnOr`h*E+c{zn^xt^e5?vvP8F+X(?cM zgqa;yHBngCnXi~=trqm`;DDwnR_iR%<36WF*Rd%O76o89(_K|J7+<;DW+6osYWliR%xru;5pY`HHqY?9%YI zjxNf@q#%cB)6M2t!AP|${Y-!=i7H)u~z4z(OS#j1?eKlY?cmDJ!)KNC6`>#gVl34KnL% z<5FH4^>FRV6&4pEjbj{&i!v4KR_QD>?x#Dj2?`pv(zOm*-{j!7#r4kt{O)Vh?ZcrV zi$SO$4`pB~8@**Km^q2aSt_UO$2kI_)SIS$tS-Tq?i5_%nYN)uZCMCEQZcE2ge2!e zxlU?S@@?QGAu&?g1i>#qAD-_YA4_|f$D)Ar-s&MhLz)XSCM8N5b?KwPTqMFO$1a|t z*i*1#8lu=hhqrKh=h-IqqeyA;I=)JN)Xm=$#3xdIq9N>>6(!kFh~jv&b9Q)qcyQj? zhTSca3OOSddzy%V#qwA6pF!=AckP6Yx`0-hBha2k2`UrU-ZEaiks4!z#-S{vz%C^bE3{{DKW zqwq+nnZ#%r*jDb1p!(P5mh8VA``BQhWzaB9+oq$D(h#nDq`V_XsEwS(_$Sv@`TrRz8rSKSw?e-j7e04|q}$;&OR5vxg?3)lrb{|e7wLUp{| zQFfbHwqjwasbP`&M0v83*2>`|5w;-p&1J;}^0Ip<7gSntDT%YdjZ?SBe?y`ZzZH4k ze8)3cE5^8d{bw%!+s@AO`M)Q5s_EX$@mJR{*%NT3X{ujNs?F2O)i!8X*M)tZS};ud zFFk+~q<*b_Lq~~SztfJIzUS^gfS2}R6mD8Uy(@s5RH+8L&bPSq^ZYzN&(Htx=l=x& O0RR89$vJrdwg3Q%YZ<-( literal 0 HcmV?d00001 diff --git a/helm/environments/prd/erpc/charts/redis-ha-4.35.2.tgz b/helm/environments/prd/erpc/charts/redis-ha-4.35.2.tgz new file mode 100644 index 0000000000000000000000000000000000000000..9fcf904342fe893b69ccfc265fe357bffda5ddfe GIT binary patch literal 36695 zcmV(@K-Rw>iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMYeciT3yIJ|%BQ{XDz#%fKy*ol*BZ+6$U-NvVhMFNAtU@({&3yOy>n`9d z{?mOw{eHi{{p^|ex8Ls<{~c^@Kl{_*`Jmr_zPbHku>B|g=Ec^ZK>yw`@jP?NVD_i} zo!csQ?ho?76#oNd6q95KE;d^*P0i1j-OcW1D?&8PaLUB1lPRX)HHv2-{{j9lOd zQ2@adPp0t#zzc|DcorjY%sC6-SsJ4mN*Lr2z$IqW761^;AY~})(HLGJ5Rzn!C-Y2} zfk_1B6j3lj3Bw7B0mTstVMf82WMB+&OfFCcW-x&h0lPINGc*KK#!@=$^#o6MG3m8% z$g$S%@oduNZ|KWzNM=2B#ZZ>OIAAaiGIW72NI<7BBA3>i-sZEspqwl)32+k3(2Qh@ zAc71Alx6df%`+6F8TmH~Szx-2lSu%B2ve5fvpGXi5R-|goo<>;TIXnSNwSCzTLBPl zwE}RC78fv{qgDWTTW64>)(ql=L7bpW6h$+L;~{`BBS|-gi!nxV^!EvW%RAZvAc34d z5lo^*gf37_Qk1piODzCel+3db34kR+M&vSy35=|76U?UbGeO8b4tGz(?FrVPy|b8{ z_0%w{SEem57!u@B{<_*h6DDkGbT> zB*Sb0!WdF1m|+})kjzq&poGyDFlBamkwYPE+HDQ!U5XMq#bYMwK!(q7%;6ES)bQ%g zdyW7AL|2kO0e}%q*$9Pm(GE!%x?)2B=Zv&kPeeNhGdMv2Cu5S$?#?ae95g2^05sK+syGZbV<3CmvV$&(i5Ef@mnWskR-p_EX} zNVaGJ>&FnJ^RpO-T@+?rcu9FX=P2vN_$-6jLW(~hJO(E%!2hcj=K#hjPLQu?I*((J zk{E})ZT{;>{1}3R@gZSH8KQjQY>A5B^TK>N0Avi<6dCe>W8Kr%cpk^R$h!-aWjK;! zDXSlfn(DnnUSKHI^3;6-j?oxpC&buht7bwo>5{-;d^TJW^O; z3P=(!L|3z%=p3Jx8i`x>%YjyGcL;v|g~M0Nd0XMlyCC730h!OvP{s#S!n2jFR)*3T zhma1zmKcNGIb~#qe?wyADT54+=P^YLq$IMGEX7U3CPWWO5~7sR-X+P-x#A#Rv5dqq z%4pAOr1yjmNZv{yTEU!_yeB73P->+ldTU;4K7W@*C_~XPg3+QSk%mzq#!Zuezr{&} zu22L7aVVG|1`@>8%QPc1!ce5fO488O;8#kNIGh4QY(Nr#6U;<~F6x5z z%Z+c_V1{5q!Amw3TJE=OD%%B@IF5NM3Nr~-{Rp%iebwjCC_(_nlz8-AMp7Ko9#wj? zB;hWZ%^;vCg&E}Ci7{oIHB^Ts-m=TQ^WJWETTj3)nJ3Kcb{Fi#F}Xw$klF(n1mAXL zcegg4fdFi&jy@B;;jAdTE1R3aa|AGRTH4_AA=AG&JtevO-Bw0UbKW(==SRbR?s_BT z8K0q?t+kpHP?U+mB~i=2JSn1yNdaPbhGObM5q0H z-U7fZfo3JrYn@(EOg%uSD2~G^3eSN6A?kKTU3-Zw`0VFumH2B2ST;AB0~fRQ`dZrBQ@6$Ny-3DdSoud z71!xz92*X6=jed99C1MhV=f~Y0fv?m(c{lG93aQ#gq)*f2*xm`hL$@?LZr7tFY_>p zq&yW1tYRDmt}uD6wT`!wgEHK=pTM(NcwAoZfTN6DNj2mQF)lCP5Ey+2W0-`TOugKN z@>a@II#0P#-32?m3y|SZX#QzNt~etxqEt5{_BkCxGMk|!676VTNW1{tKIaOItP#d@ zUWIl`Qz1O#dU;n_`k3^lkhix8qL3sPy+QZe9_G41cgkjQOMNt4F8dpS*E_PaP7izH zT?+stOBIIVEs5}WVXA7GT z0t{s{oJi|~B;fUq=;DUdF2&NXrV#`eB%aR@5ZuLe{g_%}^u!E3AN69auG4w` zshY!YsSS1WiSac9sqH3W# zCv}r!+rnPO#h`-Yb6eY6;mBCna(F5Y?ph5_lU=GFw)SR}mb)3u zVo*Mx#SBWevaneN@^w~gfuvf@*bAUI>>Vh?8@`P!!bcB&ka2<%~+&4s%Rd^e9KLS<+=yxBYnU@Bw+ z2Vf6(bk>s{%i&A#0s+bY?HeK)<#i=rb?Z)VI&LVP|a1~SnGP+7ZH)&?Kx&ooXVin}j z1_`g3(@E(#9{H)2AuhZ@dsk|22S@D<(9TsXl(o5pX&;hE=t*r$-kT~p>~);5PN{`N zRhCWyu&(;lRo}Y;gV~%iKJ9c%9RXEW9fJR@?g1aEGlect=)_hR4vwtuuTz%cWKzab zE4C*Y9FK7*g?A&AZE{Jpy|BD}&?Uja5p}8_98qh!`FbrD2ie(tov1AC3dDjeP3Cdj z%Im&S9-`fYz2m18K;`3+CKhLP{-@pR63ItG$Q{FEg5(NNnq-cFc_OTJ^N14;cbT2Tln_~k-SOJi4kka#2v6iwz1Vg+f3D*zx{*h?mC$1<_ zZd7D*;RR3lx{@=Is-q@=Tqa#gdTD7)ENQs#ZqpQo$o8;k&^dof6+Pv%~?TN{AIR4R`f!W+j;fv(|531`PNeq`D%_H++ZAux81 z4#2MPppvA3mnt5~k~>ESqFm1JIKj3|-oIiQ6fYh9x2OTGp0Xe zWS*MeqQoIdjtODb^13Q6#vBzZtoc<81gL6KEcINgIm++_j?o0|(-6i`2D#-DoT4x! zv-BtZJmJHu;vwXi!W4>V1wO;{e91_v{|>;;n>VJvKPg?MpypF+Ef|w5 zl%3D!!lkEo@st(P0$>WWXeVKKXFTSUZBZaNMuw<%;~Zozt#(Wm$M)3Q z_#)ziNQFx^Z!{mD1*7aG2&4`!+A?T=QhJthjL3yHuo;_=0hO)|M)G(AEsc%N1rEk? zJrK@C8AYVVKr=4=*#!P{Mg!G$Uobv?QJs-hB#`6xx$(pc_;TQ7RXyMbLxrZ_@YcK_Y6f=Bou#btNBMH%GXgd(?lr%ClDu;X^>sdqU*R_jCyT0 zg9XpxB$E4kB%$*e0tZJI+kA8~L@Aa z^X9cRcxZo@4iY7l{AOX5U1U26Z72@>vUP2n)0<0&|8i(dHL&*prZrocSo6 zAz|n`>>@UyNsPdEkT01#VTdSg72PTx-i6T&Cp|c4QSy)ifePPz_UtvOw+W_au1plao)rI&eZ?mAHp8^35WI*uWKg>Ob-}W2388 z?%Z48HH|nFML-!G%wOi)OKzP57XiIPC3axspG!6f8vs>)EPK!KY)%(vI9fbi&$(Qh(!i`{MXMPK3oPLK6HQiX6>U?ieSRk61@}36%*+g7WTC2-)te@wSkt? z7El{z#}Ct3iBelvCY2McWwD={a+Wbtn1bL8(&>soVK$l1)Oi63b|;(IyT)z#HV}cQ z^R!E+`F`;m0waKu3!ITeY|38149oDSaYkl&KwnSpMhkIs@qI>SRs(Vqn#;zPUg@L; z0i!5D$%WdbE^XFIKkm~RPDF_D2!$kx=+j(qa|Rlzc&<^y&Dm7~r)N0Hc*|6_%sIrx zWeED(sMa`BH2}g8XE7DN_8jGwvypf5X4GURu$bn1bWif1+gmT68>+3J5fXQ55K#dt zV&WKlHn8u(QF%@n8a#z&g%~QE|KtMYoZ5bV?znXAH_Z=^1a7wsxL6IH3^+N_j zbc8a@wNo*aVhH$z0+$&UA&@c5CI}qweFta)Q#vJ#cES4jOA5+9{9Ex|O5}fX8&<~Z zTX+W2K_x8I8svvyyPwZle1Q_qN@+&UkZlv@LDNP&2yQZX( zJO_SaoY%ETDlfAa#AvSPZHDNS#8DLxDhru&>mPck4Zi9iPWNN~b8 zFV9Kggfavpab8New!yj;IwoRXW`vs@9k5Oj0%s^DmvYIq_wD&7>wvY~vl`=x;_Kp8 zP2M-tGo`!B-7wLLRrPyy`ZiC#psVqQ)zMachQDi$i>cTe#%uH(6-ua*TGOHdBZ1h6 zaJb|jf@1kRC|8@ot03kUuEn5l;T4dn6V#~^>FI)@J|L!pmb>PtF{WUi#CV38+=>yA z!V<<1wEJx@aA_v5(#8deA;U2W;%r`MCQwHoLEA25HN1zU0)!}6ShX1%3H9s@hUfE? z0+<;uv(v7yxVx5s5dj_`Aq+{^ZcIWLgLRSw5vJ!I4phbdQ>OS^!IkQ1h61{fQHyH7 zEkT#^;9Xao^bvuZmrzhA%@8+N&8`%mAyUHf#$w^d%av(~CWRN=`uw)v2Lp=;B8)~! z;SCVs423LS0E)!ipGsv_WbWWUE^#I_a2eCSAyDPAFF!2A5s(f-PFxgW;Z-wo67&N${ELz zInW@&^9_Gn$Bg8MmWa*PiHz!;YhLh;2lF0Z1ncY_o z*_;PgyQos!j$x=yi|~R&(C>=>mEB1Oox6CHJbkKiDs3+DDwXlMlVeGPtM)A^?3}kw z2+o)0bgCJ?o%biN!8(`oI72DqmK`UU@*x@Hivp7APFNsh?PFP+f{%uF9MUDpqK``a zP$a_pIeL1-p}g1vUCSKx(>O!Egc=k-N~K?}gtk@b!$dZzbd--A3Ehew%KW$f%aWr2 zU12J`Z?tr75`uHIFx$6hi$beEqQ%EjPn>>;W^Ub=4Q?SMgqhP{IN?zKJx@ZDcny5) z{zEQm<)M*7ywX8W4I~IN6bU^_lSg{nt-~o&S&@^SlZji4TFo>3V+I_T2;H?0sy6}v zy)e{B3{9QD9%bLQMqy7v;1SolFbz6OyqJ$J$}h`BS@hR68CIW*^3nZxFYd>!NB5&9 z>GRL%eeC}Sc^|jTSii*yp3P?p9Glfh;gBjw45n-2+)<9H{AjLBbV<`5r zpPII=uC9QNJ>Wl{3JPd=XGooLiSv`(ks>{q!7HiuQiV%bR})UmJ2)_lsyYC?LLlZx zFvD?-Y0mZie&4jK^U3Pd8W-BJX;(5VCoFYy)@Sh2fZ`G*j!`VUPG;8T{ngDnVeSsn z_aEWzfXB3e@7SB<_a#ZvU2fY!RO0-A|s#P zb02Pzre}Uiex+{9bNI+QdNhzT(i}ZfzrE*xxG<%3E@HIAp+QdM7%>@aK>=nPCVEqf zMgCqS)JS9$s}}FZtaBz~=n3akr+1&IPde&M(Jeu&8>w!@khVk#s01t%%5k!^RzKWXo?L|!p7#09GUeT`Ky5i$?50s( zZ8AASAw*by#-~hoeZ;3g9hjy=aDAQUqH3Gum3*OYgD$waDOSp)i&RuB`b+JyjHDzc zlZEg_x)WeZC=(|O?vV`#eU}eOq$zxYvRR?)bp)#?nIhWCRBKMR0_BtoAc)8e;-t$( zSJ$QHZzMsIx1ZcL{8hIY&{X8(3klc!#A40YANg@4zU}6piicu!K_H<33rcnRN_Yk6 zIl2U7EGL&*?bK<2tI;{Sv=gSD=r+!gGKG9?ITXw*?Q2U1=G!&hTPuu9VH>pCLa zoJk`Hex1VwAN9(+M!fay(>fP}?)Hm3i*TV}YNr8Qngq~PRvB;PM($S{nNzWq`t4wI z;H{t!AmwRviWzj=Ix2mM%9Ueh%E?>I-#Y1Y^m@-1sNQQrnYCSRf~iEg{VPo6nZ^)j z;d~~7i!h~gq}B#;$~Q`6@JTF{AF(e+!f>8Zu7!mt>8FpO?iH7!pw$?-x%Li~fe`yr-hcj2A)L`aSI@sLbeR>qgnc_Ay(W zCah?=wPaNVJ0_=q{O8Ra0(l3MB$RgF2%}mBHR>@Iz8ez9$}vgC8i*Q9IY{!jBL&b( zdCiR|u@{b#3vdB5@bP$m?_hMg|363Xj!*WFPe;4Q2S+C#yOMWs%8c}Que4T~q!ofU z?{|WcMEC+j?FyZ#M&?u`eHS4~5~(1^SdNGS|4CWCf{A6orJ_SfQj8*T$j6<7>|? zmlCNATU3BPRY;u5Cc5z)mYY{)G#b1Qf%;ntw!Dh48#2V^S;7g;2T}Mbm4>8_$#Dzf z94)AzjPm*BFuo&fTKsI%&Pf%EZU`T(a?wKY`iK3Koi}edTdD)|3f5ba$!kymJR@_a zCU4GLB12*6MgbkW+d~&H=D-v{8Ivd*0k4$*EN^TliiG}H#9f@qqGX1^6w)cc35d>w zqx6YT0z_OKjN|q8X#dUr?n!$CJny(If>%yC)!p_6XmhXu2P;peQwAAr>)Ln>GVs5z zK!Ri8p;WK=0G@7Ka71tlaYom(4v2{QD#oje6N)mnF8jIxE;_R*?Zs0XmjA?y7?XTqdV!EJCg2Izz!38WU@J=)h&&w>(FNIGrk# z3w>?K&!?v|#NcUe!knfgI!#D~PC1+FdO&-!NHzq2F)uSTL03cYul4TNoi+28!EAz< zXksW_AtpiN6BSVnrCl%tYXkEhhC2C$_^$i^R`78HD#(3nO@s|_HfNeLIGG@t?+wmu zkR<|XJkAkHiyeK+>2$gtK{O)?MeI|a!|YQ^qEALLeB#~yWW}0xy5Bs_X;~ZtUhnku zmzLDoe5T51cjs`VG`{FeZO@BM1bB|*3dm=P4AT?mIwL$5&LO6D?~Sj85s^M5Ms_NI zFoSd&6O#I4i}*VJ17c2=OJfHVLVi@)JclL_e-6Rd9;;v_fI5w*Hf+9j44YepJaLWI z(x58>CRU43D#0vtP34pICVQ9Zbz0SV+Uv5k`*XB=GPJuX51do`7FjD_fdO8mh8mJzL`;}B0IjcdY!&(T66Ap(HcWekKO zqei16?&cE7Qz#K9oDkUgX#_@FfMj4adTnlZumX=bW=~d(*j0hLw~A(N7Q|dTM6D-{ z25GfWW`~j=JFQ|2vL4~O5M*_)TV4FIi5%vCv|1_3D5eZ0Oh!Qz-M#sD6;aVDM+l0& zQ0W{OS37f|q$^3NU?fkw0g>=uRyi2!VcF1!;jHTb{@Z@_S5rfu6$skCO6~TB5_~48 zKUySkhC_M5H^n58cASJZPc&n!TZtMw!c#08O6GCA0TNPbOt^(j#1dCi*#-!wgyg6G z%muTmWt4RRSf8YuTp!>0X~cT~lSsIx6AkBwANM!JocT&zEIVQ_JnvYM78V*qP6j<( zs}6t*=zm-Ja^eoRKHV}Sk?;t}pKtju9o2A*VDwXl8G4t5s2UK(zlmT9{2Ld@{U31M zu`t7H`G#vbn&Otlr}H=~?ZBJGxs?CugeZT&lu!}pBbwke-4HBMoGx9R+Kv&m zC_E98Lz$}@N|);}-PRNE6U-9fpc2w=h9^^&r$@~vt4whve8?vI6sFVD*G35<1_l*gcA8$ddjcpdw zE(h>Ej!_!~#p}p6WQlXMunVZ$lQ=9@AcVWsK*WU{JAxvja5NRycT&4}G_|r%sr%bj z)e+RSZ94Kr5kPCe#Y-J!NT_h@a$KqAAw0G&YdcgoLfZF6N++&5)~AlMY4$R=#j%(c zvu_=$VYMlvda6fYs6E8_9+<9muCAPGiM!_P!slpFI)kQb?cwnlqu@1);~7lA`rhk~ zfP}8pl(G*K;242*9ps>ZM^nBQ*&}AH9lDj>9dD-$PYgi_MV~|hJ*g!Y@+^w&C6OU9 z5*}|^s{1qD`b6pUm#*5xQbVByZw0VWZ4yZ_70qX9H;c|rK3( zV=jVfJu!MMmzJV#B(L(cbGngn5n!Opq9i1dkR{;!CCA`+|Np%|INslrF3K&9(NL%c zPvs(fdX5%P1rI)z&C3_!jDfEF{5 zLUd|X5v}H)ZJY=zE#~VVPQrLDwA0;Vd7bjpfIr{u|8Q^!jz>G-==k6tJ16_#$Nhhb zFCUU_w`>1Lyxl+C^Ok;E(IW>xF8}|o?nvnIC9Aq)Qh&CnAz&7IHzwIEzmhz^Gz-i# zJOoc=0>)nNXWp&|h1L4_-+$rqf5=7H&%NmEmkR8QNN9j%cxL~2xV`n_+ovsEr&aJZ zRwfVrjRW+m*fZEYL^Zw^r)P z-N<|_P0Cq^7S^@!`TpGT+*~E0H1d{GP}O0jf>{EMs<#0s*|4uWl;Kq>-IYk$ouWi6 zp;{H>!_?iK!mJ#wZG~8bBtwMS*OTcu)F(3JL>q|$VJxG*Qm03D?3c7|LKB$c*_$lT zqgxsLWd+};tx}VBKpg8r401)pz3b$Sz3FKbGr1)#og)@Xe4&{qi983=ZP|Owy7p^w z3CNhCM5RGjelU^zS01@D1GYYfaZE*`D3P9d5|c9+cU1vQl2g6a{88-VO99yFw(LFV zLEq$m7OB&vr^VQ(>-r#FVHiE*=Y3}f&g_v;Z*(vVMq9?qU0;SB!^KS+GnjO0s zWo#$|P>x8nM=ley)rPLpOa}g_lS6WroX*qhf0MIU++vZ+ksVp8FO!$irR^K=>J{i4 z5Mz=J!Sk8<9a|~=WfK%#piG1zhZz=EqUXDumLJ8c5g`>c!-c1eq)ABxuCIknwI-YB z@>e%EZE$lVj9R;&0MPkt2D8O#)mufcOn#_G3O-31oH%)SM4Z_{f|)slCdbDM~QeAm$x7xhHC5NcyIhwj!Md)5h+w_@yn4jGctknrJr~sIgVI7sM2K5O*7Vvt904Y;{Z%!5 zypvzQ7yz7p7nATjPah{DbYn7)xW;jTBjKrKGUQJ7M8-nKI62p0kRzv@m1nZ!t_Bl$!RP%^;@l9PQ(E(1nn@gPd*6T1NmgAhmA5Dd1z z?QTAQ*6nxuy}`B>yg+jzYd(9vWmV5v?ET(R)1rbKX&PhpT?TQohnVzh$fxbFeEhDeR5h?)&oN+kv!bS?#^B9GWqwsBuqq&G>3+qNmQ2mSu@&FvS1?LYMg z&z|)+{{;H?g+SdirwnF)>fgDoV(0!Kk5VdCz8K*vLXkMO!A-3(NRUX~2%K<>E*FR$ z&<8g+;vmxXwdXsyxw*auDB(s7P|(~`diUq%CQlq$0+koap>c_s=cQhwf-t&3o=6(8+@f$wz}h-r*+UkC z2{L};COrWG0quWNlBAGLUtM=2$l%XUIR;OExoLw>;0#i<{R{*_ghCRbPOG)q1-p5F za`i>!E8=FgP;7Kw4-vUQ0aW=oyWl%>;iRbD1kFIwD(DOFCNcdyQ(*#Aq?#X!Kxs`3 zbcI3?;AW)A+X#}YaPdmBhk_Fn~KpqL`X4af5YK+B;~N9G47#AYoahOL&<>1iWOR+(pl zR$|_NN$Qqo{XbQ=Aaq$8-%A_y>;H>q&!2A=*8k_5+uM)p|3f_2*S)X71)dFscYKUv z#1<)fHREInr)UVi?%mwT6+l_oZFM1Yp2oh-dAhJhsIOK|FIfn%G_EVu1)l&;B9t)j zOoP=DMtELvrXr{0wo$GAAxabw^DM|RSiFG-VhV;RV}S_u=;@!dXH~QVg6w+dXtAuC zic6`$z8Lvq1COSG47Xm^gb#ovohq8mBT*bmz-GiKRrWWCe$9`RiE9`NV_~JV?ueAu z7ApK(ppy$G#MeqO&3A=XY9;dqH#ZAJxymM)Fd;^LYd|*@{^MSCW0#d4mqx`iaV{&+ zx1n+Z^JPwLpL*~p{FF_tiEy(ux&JqORh)`urdpjj$4PWRL7T=nL~UL7=EitKEa=X+ z>ub>*$5LVIRI%_gudJEw*7mpR?{S5?3i3oU;&hkSH-ZF)z%76OA zrIQiSCo%H^^+MBM?yhpPX!z8MPH}mFDkJ{(0kR9#&yrO{a$VU+sr=wl*~v!#*#xsa z=y#LN^+K?iF1193cAn0)Ev1tyk*XzDaXtdE@2p%HZbgsPQMmfI4c4j~Syd$FerP^S zLNYc)LaaYcB&2UBn@VMqss9$DH^ygOI&BwwdOAVuRQ6`Q1FqFK2ENI+67U%O4BBg> z{lk-k!~Hj>NAHeL+8{u`fn&H=>NA|TwxlhqrGN|C zMlv?`{J9@IQ*Sq)|J)Cr|I+!;?Obo&tXiw{ zfquQ#`DC}b6MPSY@h{hdjb}H*&h?9%(%VnNPUn-i@b=C6uvB_|UA3k#_z-;>e26|o z63f4Rh&t;Zx=YLO-gT;5etxsQUT(W{J=oaf)gA{$h7 z*Zqynn+L*fvl6@K)!1!46n2A3?6#}1d-gf78&)Cmq8gFs_m4 z-?Y634u8%v1katfiVzOYU~PR}{_zdKoxHV(B>6ffHifM=Z6e}`BRyUeO|-n;fhvKM ziQ3~+d%NouYf`7%Z5LnwwF{Wv$kSxQud`vTe?Q|wWk9P-Lm&$x=K zTAe|Wju_wQ>C+K*X&iy|50(OSuy?w4eQ>z9|341dc~$WCpQl#2SM3jLzG~%moyOL# zT`&dyNOQ6%U8u96E}PwA&vilY20ys!e0tpO}z|!tkGH(s;_l)<-I9u{lVrvD3CS;z3D8MOLyWTlG zeE0rvcmMQ&tJ8n|D=31zwc|!dWVbf-`K}5St3qA)?)|>6x-L`>WD^|kf?osN zT#2U7CM^?Z=E!qFmZ`zYnf|uZ^HQE*k$1bBGuA6jS4AWafs8Arqo7S&zb^1{0Z33J zvk#U7YB>X}(9IRQcwgOH*0u74cY#&S_986BfFVg2DqP3If4d|VDw1~WRFYti-&e2b-m9#H78_h zbV%#+e4Iq+iZi2BAd1>s+h`Wx6=?VC^tX|@VMvA3LBs2Lz5{u9Ky|$9V^@u2mGmr3 z5?iKmY#nASER>EuS6^7;G86ev=NT&Sxjvo*)$Cr^(=;J~B&cGIroDf4zg8u?U-#r< z>t{s2O!-04*pVbSt)k>vjFTu{0I2VbThHBZ`6TX(RA}}m9kAFD@=G@3wy?MV{m%P0 zC#ShdFhL9@>~u;fqo5By0eE>1o?e4A!wCax190=SKow~o{KjeEf~&FVWY6}qjATRb zUf)$>PWY6pE0i#C$cT0dBoe@`FjH5#>Zf2KYC;Nn@=t5__ zQe96LBVq&>seRnrOc%oo%P}ZUnIiF*&6>*cK5ZkF`P}PvE%T*ll`Tu%y6Y}>y~GW6 zA=7*cT5J7(F=Muinb0Km9D9rdpsT~{`zzJqZDJV=SXzK`gip1@P|s- zB$47po)$md%@YN3*UJ(Qnb&8<@p83^F;1dtZ`D6g=J8ck+5t^1wp6V@imz%F{7Q|w z3P=%vMc(pQ=AJ6>U$ui;bSLe?iK+P-R{PVG6 zVU55D*@3)hed4tMJC>}q?l6>x162mK=ol3Pi`73yO`0j^y@AgJ-cK)&d z6jU&UkEzXl;_`YGomI$zR~~mzrOD`QtMZ(k?HO+ob=yn7^s%U-6=lVW-2V`q z|Cx8T9~q>MLB_b{5z4PgEV4*(WCneT!nS8Y9`rpme?mGnh{$ShEd);~*S`~T*#$fF zTr5>hHh91tMf~TI@3k}Qb4;tkz}0Ry`E1E$`%K^tdrfZrf~(Xv(>zWh!&GHk%4>50 zkCEmrSTjPpf67$e6$~rWsLJWQArLaO8*iGbFq{e#eHC?Sb=o$i>r$Hf z4A_h`@|eRY5GkUUIV~+GBsY4Vswnax%`g#Zs{2oimC8lxy{xt;#z9*6`nZ`i49q@b zZZOGoIj`D27tz|asB4jmLw~jAxL7zz6_lHHr(}olG8H@Lx}8&;)@+~WEPzl#cem_{x>CEOG-&@io1#i7Y{}qUc|h?}&CK#fsVbIo;jsg> zKEGVXJpq>_JE!LE2D#UTDTw85S1{2UzTfj@uj&G_9wlpocM30%BTS#JU0XkIo~mo4 zh0J`amln0g84(rw0y2f9btDodv7XnXd@8=J_uYT!$`0!P4^0f08Ye@BGnC~oBZ|&M{ z;ikMtR<%{C`9gQrHl!PL+BsLI`t9$v32Vx`Z?|R zg%EXB5|d?kGC^6Ot%CrMmECF!P&MVa+Jt_sNwm}{e=jC8>S&mJG1z@`uzz@R`tFD? z2d~<1c>W%V46ZR-NlQxg#D&UWEz$`2>hoRXAJ{Vq)bQsI0%0KqiB zy1v|M<>;1^fQ$ayHPrb$R7mho_&W%B-JS#yIgBWnHuXzX(%a(PGA z3Q8^*zValN_G-c{%|A3H2}LXNl=u9r&0NdemG{VG1rbuiD@9{tMOO1AOS=WPY0~8W z+3V}_NSZ3zXw}3-nMp1m5JF@dHNXa&~*3A7WI5Qlv4JT z_Jq*y^`7Q$K4iU~?XxL*H$USrs3Eumwv`bS#t@WD*(iZPL$E-l|9w@8Lu`Lu4w*eb?(I%g5NOt>NU$lELPb~S50 z40$DKSCEe-N14)!s(fXZ3gm>NuI4VYTq;tL+R8VZy{G}LEQ!BZ22;=$YF@t$PWF%A zTBvh7#-pm)DsDO7dROOaH`)wX;(r8HK1gfgNkJr{=hiGr`Zs7j+M_*(-@VzF%CJ^c z>o<8XAy(Jy>Tp_duTn^6{+Q;*gmSXjyM~0HD8=s;{mfC9d0vGkfGH3?-vH<;MWMLj zMZH%J8gNL=?Fdq03gRvs%}#i>FOeeLv>wsT}Vw;Riac=X0NMlRF){$oTBi&wxTGY*Sc6->J6p{ z#%y{jzMW1POHb!ntRU;uk)FXhIJ($2m&D)P{5%}~Vw98N%a<=-7W45}RJ?e0v0YT8 z#CUL2%x`hDH<)J_^d!E&sZT^)@h1O{NWx+=N#=2!yDrwm#p}aYxo1$k9na&q+O6o> zAeON7cAKchz&ax%aiyu{vy~+}PQ5$&GOfIbnAe>r496BWX7g`X4DiUlw?^M}#-?bY zKP2r`4C7hz+)vgjpRJNzUFPaBb3h=H3}?D0|7xZrv9y`lI7pUV>vWS z+^#tY%JFM+#TXQQuphDyNvXV)gzCy!@Xfm)t;QX0D>QL*aQH(LfJg5RfAB-l$bG0v z!1STQE!v>jYuS$ocn zmZ@g_Y_yQgDGEx986JacK(mWe5gNU2K6JnhxCY4?%`VoLDncDFL0}WyVa2SKe79Oj z!I;mt+S1?i5;r#>!xWi~%=#qe%j8;(RCt`He~5YNOjLXH9&Dcer%j*bvuTD{hQpG+ zX*2`Mpg9Frw1@9eLC|R2SwG(?#8Q?7R}r0h?;_b%w?nqEmJIeqG9a-M+pX-MU}k3D}7u%%>5I&BbwITD3eK zumO0yQMJkOYush0#o+GOJ4eUw{^y@|487%96GiMSJoZP>UbAbre<#OC1KBVp8Hfo< z?L-EHzMa6pIgqndZh)`8a*`9M6OPu#iDum8pv2~ih?!o1^)-%V2XwpLHgCfY6*2)o z)^?BvQaVwzwApse0h~PG-P2UjwX9P=HQB>kj+F+Qhhdq0NFDfF<-F+muodyXE|dBu zu+Pmb)sPrppafCs3Fqa#U0-(j=wrNYIfAkrKe;3#w_!Z4Cs5KGCaK47$(^@3kOslH zzWB(y1mczhBTfDnIZ5>yM@?KemNA4B0rL)EJzKf1_vy{MoxOv@AHME@zv#&NuRCha z=~swCd9%ITDqk{7xb(u0)8uiQd~wr6?UuSh!76nP@GN8^8&N4R*jNvBgDD!h9`4;!0&5YR_`i z%FX^oRpY*MW6`SeBB01~<|@aw5FqA@Shot)cHeMxm2@{50v^?%-$A}t9wJ%s z2+17D%M7|$@1QEwu?cl~T390z-cBVJaV zFHTX|>9z!ZCBq|_!mjir1^H)zb9Z+NGbS$UH!_Boe0PDe3`Yojl9+Rx`JZ?;nvci$ z3bcb97@yL?AC)e-mn|r}DfLG~4qc|QKqGo_5wCx&CCUNVct|WfReS?gP`bw`Mvx*| z0H8o$zuDmGyj>YV;rAMY_Md6+XRbIn&}6%-FiF+lihGV_+BLeMA&UW?!3nAuysj+M z6=m{v)-p7~lw}L;ST%%gMOc@U*tt;07p^$$}$~l-s-~we7lcd`^U?3uhf*DTm48{P_ z5T@FRJ{PVlTTtr3+P{LcG)92TSvqIK7688ya*j}nvS85N>~6IHxFlJWW{A>YtGn49 zoNo39FZ!EJ{lwzd5FheREGDhUuvzSXWEoCgIZ$XN2D z8ekZPWS%e`j#e$&RfQc{{;*2OyT!VpY&|gMo0x5#Ta+OtKj#|GN|=bc)BU^`3GG-*QgGOX?mBTGcAsnMKl z6ovDQ*C)Dm7ZZnAqI*lJ(K<#+B$9Dxv|=vYB=B95Fm%O0gi@45oc)MoBu%9*_-)6P|>szXE1^(Ly-nQd^Bwu26qr3{J%M3~3?lt|y93s__Hb`d;hYf;v|=3H8K@fUbgXr*_cb_{tcDYd z=^OAHPArZsFrOG>?c^#B*(*S1n3Z0)$5crDHfU$_WQV?|DARwd4?)>+YYl#vqBl61 zU%eA@n7W-j7^^7>6M$wZTL5zQZxpf(YqDVm#u=IESu>BB%F`$C)21(GK>R`F@+ET; zVoEuYZq;uWZMTJ7QGR^P> zj?n}~GR(?MQoiK72zRZrIciNi8(Mx4y!FpyMMx2Tp`l;gp{FbP$O{$ zJ7eldur`P&x84)P1Y?Uu6eH@DlXB4)AS%~1r;N3VE#24(1a8C1U9A=XX+{_cNjwB6 zyGJEbHFSP^+VP}mUAZY|Cv91moK&T^@72=Q{JBx<8&rS)-agOACx1NtfB8ym7*P~v zh;@CVB#U#YgDrRFKTd{!%Z<)6KhTvk*I>$*g1hQmH z``(5vAe<;wZnaX7DgQxJ7<#;B;bwhN8u{Z+-=u^ zCGvl3`&r+W|C{~i+uM)w{~?~$qw>ubo)nH;&Xh%_wKug==>Bynr$U23{vZgdRtCYh z{-vQl;JmahSY39+WFxsPyR?ezb`A;Mg8+9|CgaMXXm)|JT{26#UEBWeSA*`LAN1Q` z&2TW!z3}pvLyf(LNhpke)RqYA)765@_Y8IyIII0*IW3p8rWRv9_n~lF=FqrX@+JESqrjBp1O078fsLKR3+g0JVbrpO;6s9B+=~>$4)kn<_v-Uh-)L@qx-jFL-R-p8{?6x7+(?HrDsuIa@6T@V>p{;i>>7ek#jQibWJ z7FCbO=L>l1$hoy1ijKw%3GyiD(&VvDNHyX=4O2t0c^tz_sEA+f!-?azo z9vIRRWX*U}u|$D6C6(^Xo@&=B=gq?YaV^t2*(s{YpttzzNKyXt_g?;M^}jSayKCsD zc$V7#{jI|O&vyU$WB!K+c?$B%bL37PQ_dATzZh9|SkKZKkIF@Fs~D!y*?*YBQ0r0j z!IY@7OVY;$ez)Xf*ybPnsg?hEZe_7V-gYfmCjU2|JuBM(&$j!I^8X>8qWm{Q&NVSx zZM9bbbz>Gc?L(3wLPgpK<32uefZRSXzaF8l+xT&HhG#R>W@vfE5r%wK7RbCE7XNCM~3k|O^S+V~QF2`cfq zdyK}Mwpzf{3{VRIOFpkO9FKb1ANr}2|MzqHf2sW691KeS-|hb6{{MqKcFG@f5Q{RB zq2hUebAQyD$Z&ToOO3X;Ky3euZVh|A?l+(QHIje0=}GhAqt~*0_nW_WzZvRsEh{mU zDE>W8qH;~&;ZcbzrPHodm^z(m5LrY-}`o zefs16KZoMREMIAktVw8abJH$v@YP__j146oJ3oz1NBg_S`zNP6ySw|NQ4LmGUCCpo zCEO&O9Kh+pUL_0|hKSNK4F3k^XAe+*d_L<_zy9AT?Spxit^Y5I{{O-Di^u#A5A&>^ zH{lDh(5eou6e4Z(`=PqjGb+ll`(LS6YCz4g+c!~)SiJV(cp6=Gq^(@g9k`qaM$bwpD%I2U2_(Eu5<;B1cWzFfe7jHBYC?#1+^&( z8YU>H%}r1r;87?48?65-`}}8%|KEH*D4qY>+S+_v{~zM1SS?)NZ?!$_xGI~f3zbgG zN89*bY~%Vdtv~Tltr&Q!*Iqv^-8oKD$;$idq}{4wbzK-Dt+)7(4XTs>bbxg_A_sUy z&PrdcLJ47ZTd@{RZ^BqHRc0}|_}YIV=mR2B!8(P3@*!m1cB0F-m!Vn2vtl@!A?(Lp z+>&%Ok7Ef+FnjTx?>u&?ULKgz|AGU#Z}~J>|5dct=R5!R?8Tt8{=azkxc)!HV@p|E zUguc}Bon+8l&pC0#f;4sf9DeSo>rdOSLSirFu@92$@Y*M*_Wm>4sP>g1X zWs4)2-}0M6Hf_{)srD@pz%xYV#>}N*22*q*UpB)b2T*8TWYAtd_2tO!nAq8)jHD>z zf<^<BF0@tIoR@$bEG-_uj8|5>>#KVSUc_SW-a{_nwy z?H7;!p9gt9Q$Ra`MAfw0fOezOc8GC!7~qF5bcfPS6Y` zOder(PtBCS%Zl?ff((sV1{sdG#%EF)*<6QEPL`FxwsXQ1lvmfq=%mTu&FOPY!J5>y}D zvV8$nrglf(4AjtS?*J*Y7>sfT&6E_Q`X)eQ6FuU#&Idw1voNk47y;=)3V2zWoLXTwre~Q zJF)@ei758hTXJdhLXL6C&G1fL|EDl>3fc96xzT%X&S&>XG8axJ2a|-D*ZWr}oO2A_ ztU`eleg0fTpgtcFSXlaFXfXut9Su`L83*9y1vNYw8l&vi z)V)M_GG#+B==Td~@o`tfd4jT8nYc>^X6ew}esESEp1TduibGVGz3xv(Y)PP_l%!e5zmWvB z?FKKWD0!by$S@sa-uZ^jmg?D!tGR%7lEo!lIPF#PZY9sI&Z~`i^ftV?Y)$m-Hp_x{ z+;gs5BMVpP58d=+cfESPctO8&ZSt97b2Vv35l7^?m==pl8K+54QaI6_}kN3dFBsE3s z+WGNt4>#G>O+jnK2K}P)T|osw%GZRl(ZW&eFGxI}p||`iwUZdCSV}?#iR@Xg z$DZ;G{vkLSB^Sfh?@bMwU6Wd5v(f(H$-&|No7026Q|sDO!=t>J+-oZW{5~Z7{~YzJ zA^3CT4f9jC;&s6Knxlnvcnh|x^Alj%JPE-wUs#^MZkq-`l-P6lfJEeU1@n92weh zXl@JdKE}vBo3rv@Ipa_sHAD8=#YIZRishvsI~|`AjNT>jLfvFrgt~0WDvWULfhh^s z2Ek$)geYUSN~sP!mA)u~sLjp$IfQNZRFw(nR&j}?Tj4C~iT@5oBnA5eE}!inwAGfj z4&e$I@EjM<1H8Zomw5#lzyyhE&WIZlXJ{hNQ3Qed(|;h{l;hteuLr@HWFZ1iD~`bn zl7O3=r;`18(R`M6v*_&9Mf0n#03Rk0(52+^b6SVRz;p*0(Cjd5hE;4#-uc-U=#pz1UbS ziv;iq4r}%)$q`Q=!y)XE6eW~SgD}P@VgEToB%6U?%$GCH+$fw95FqfOy>_j#Y}|Zk zgTM06#fcD3XCwm8`hDR0s-@3Lf=TSer69m%kmd;p;3FIg-^Y(=Awu18GI^Dd>KfeW z`!OMx39m%I$1!@gtpK^7+5!6e$X1A%oXmzZ;z#?!4UxpTHzTDF}WAPt~z6lm&fP_Cj3Mk$S9h<4%_n9$A-A9^1?^wzsyciOeX1=?%ZgPV3yNNf4^UuCgWD!1|qAQ?W4 zF$m_qY7TxkHy{W+BF>BM5^?*G(zc(>oATPz-BA)m`|WI)y;3CDEQPxYQr( z$seUjG==0AcR|s)NUnG?%@CcEI9dvzQ7hC7j z)m2g^YSev}gk^J-5+jsTC|2~Y`PQ9Mo5#jK?TkgfXg~8m5SUY7`Sm{z1&-{OICxOSz)#Esf;@ znN~HI3&>Pk%)WxRH=3QJo_{pE`Fe%4cV#>JG3?0${s9c^>n2HUA48wMxX`Ck6m`AA zVnBR8KSf0ochL4LAl^>nuK;#yt-k`^9W?(6h`*Ee?>pPSf`)85^EsUtRgK&S&)tg) zqYB#Hx$gcP8C4+v;C%PF92))x?t|m*!L8wkb$4#NFXx6I;2(t3YCRm*EV>sbhySXB zyb~4nAgJ{-c<(=`=xx=!@|g2A7agpC)7{)3U#O#^8rVvn3QKuy=B%ixa7TZIrKME3 zEQ&shJOxoO##dj6>!SV)DM$G`j`1F1C=_>v-XbcgHZI9~T&tY2>5De`HExe5w>)t) zk1mYIlUtAOi!b%$))%ZVG`-!exZ9-56<1hqnmO^Uu)vy+a*b^_T4eLYb>4Nh>}GG6 zE?p=t+VP&#F91Q1r&fEFXI86M8^8F~)9#@h)Z_iVgVE{E-rjM`opjy=*)-lS+qfWK z4+V?RVXtRhUeb<92;(WCR#yx}7)2SPw7?>joL?K220TA=S~iLP4!ZoN#`$o_i@=d<%w;q#i;pKd&dw?d_2idCR<@@90p`{rQ(@I>4>{Qbe3eJ^fiDd^cbsAAeZM~m-q zT-q*N*4Xa;@yRV(<1bfhZs*ob!Cg;dUOUv;iYVJ@W^A02RgRaUpScE}TeKQrjYG(S zwVFdHr~0zX6K$$jJ6OE8HiAlpYt^}ED?GA`68H_ze4cO8r$qvA2*CD&)gwlzO zf;KIGzJ=+qq|Y|8B4qs;Ok3W8UY>iaGRKxq4rL>jDwZ_GO1*B%=RqA4-<2rl+EoR; zNF@GXhGw*wOH-_~o(ZRWURvyy^}j;33o0j%)PX#b!% zt)~cn>FDrIFFy@hdJ(wJ-237c3C_zRiWYB4godF1;`wv02$O!`TKT1m)CjKv

ET zQ-%^I>$+yD6fL_k#IqB9$%?$Is&J_)?}fay;-~;nR2JWh`S(kfx+fw3a+9;vfzl%gcQqHNH0r=Z>Qa*`Z+XEtF96vI!J*iBSdXBD} zpT_1g=yTeyC;rUf)h*|y<;AGm3z%Gy_6!?% z?^?R$%}T1dTVJiT@~xO5nWZE_2^)gE?PgBdd$=sKg4p>K+o`>qyBVeppxzeu!p)l{ zJfcMtV`t`7`szG;+n z+{<(Iw<_Ig3IWf(HMi+f!%XTGQ)x9O{o`L&Q>k$mZ&K1-mY0FjiVKyBrK;{#a=d3t zE>RNH`S*pCz#=0zQ`8{E}iEupe-vXuzoZ7d6HMw0&18#D^Rz=wcm)h87 z%oA{kP=q3J69br2&e`*1Mj|}MC<56T3t49xsU=s%n-C0_U<|P3*y0J z9{8K6N7NN3CBFE)yOlXgt6p)?_-2BN;ih$Xq}zp!VudoHw8UJ_I77F!{<+O4=O#=2&4Dy4k9&ucJx1)bQGc^D1rZgF8v z0wM!h$Yya*T%@xO-$N?<3Ju_H*ML<*X?7i0bz;Hdto<@4%a_iP*LOPJ@8O!q`1l9R zp|@=F|1nnn%Z-)yf><6dmp3wHEd9Za)G+ZeGN#}+_mgm?)kv)byF+;_M+M@lLH zyq`^Rd)Y7JX4+c);An;C*N3et5;Vv5+d+R&#%gy>%+)In8<^2)>6{ z*efb3Ch%+uGZY1sgy&8mlB4R)6H~up1Xx^4{S?i{IId77J<5!Z8h)!wfm@s=HCrL| z`L8N{ZNGcOw2kQZ!Fw*KxaL_*2R*To%f2El9~y17qEYz zP?g8$GdxSEj!d$ym3{ZXiPk-yhU{Hwv89W;d9^-!=;<@(RK3s?aD!x}7 zA?EKz}BR}L+zSk2r{m%?N35T~;Q z@*QJ!+>j8*q2djgN{SIvx9v!Lr&uYyRi1L2Wl7jh1NZZ7eA`}UwPdQ3MXDo#18|T` zGDNBRwXc65@rN$Nh^-WtdS%qj>fNIic6}Q4R_lGX`RA~QtRud=tg)?Y7rSL9n&}+4 z_O>Hwz%7~QTegjw5k^8155fDrqozeqc8?76XhzNEaRUyOBXm5E^O$hAM;zq3aH}Y# zN&xQaI5OI)j1ur9IC4OU@4l+e%*BbrL*v3lJAZ60a=N_Qz$v;EM$RSAy^b9oJL6Wc zr<;vzX>^wnORl8#JaTvKeX0JJMrXGU{8rBr{ckXM@yyl#1}_FL9`(Nmd8z~9_At%< zfA+q$xou-ha6jW$^wLv1mPQ2iu$`EDlUiBUja6|RmZaqFEyrs%fhO4zh{o-1N@8N~ zFPNV+|7rfh);aw^0|fYxZQW!?tg=kvaZW$ayU&>yAc?$~rxT{};8Q5PHPeQMH{a zkpZlu|NBp#*6IJzv%?4ae;=Rq=YLjM2%1BW{b}kEO@eUbkagfKI@ePn`ERK02Kv8Y z4(B3c566l&U=Sd7s(ys0iNBL#UU-E8&vVvJuHtKj;5nXRzh-jeZvgxKtg_|9=a2u{ zi2jdwp7MNhS2nPY{vSTuub=fp%({lAaTADaHZFzfDt4Q!$(%tybnL?b6KBUnA5 zvS#Sk9{vG3U_+T4qggSJk;`O(Z2V5M_P~nn~0|D=Kb?>{?wp#S&r zSx^6KNop#z<}PtV`l@qo+{T?o!@MH zy(ziFXKgdP)lB9zyOeZ1=1CYb-*{_f4^{yC;vES`?*jGFbJ!Vbc^bmItIkNQgtuSr=scIIjc1ONkclGdX0%d(a z8pDMf5~DW7{DyY_)}A#539s&0B|N#ZIIHf7wZ4e$JP{A_{}=mgD*scf<9~tpzk}uc zUk6W)9{j)e^0~8hXG=}NSs^x67&b^oayQihsLzy1uKfu}_o8j7BT;KX(}H0b};Dkzwp_VHF_ zNOpFXV#Na?3rmTF!qpuGtx92%|DPhhjqmKog8pO9A~!Jg)Orm?#8%Z9Gs6NuRQw&& zq+`mGB{WW??vM{;^y~@Cll6|0?ScyNKu0a3K;GGTh-&e4wb3OeWJYcQ;cYnW5|DQem>Vf{>$ESh*R}*0c^ndw@ zb=jr1V1A#kMo@nggO*Gy^ix9jefc~0&^2-fWX z!-J!*mh6A~5Axr=d>XgrK%{rwmwusiVFBIO?p#X0(3!Gq7OQD*D#TT%PjEG>IaUGl z=AcT>^@dQBTF&)aYJ@Q1dc(`rgzL3;tC>1|ilQk?uF9;o%Sk%@;9fGN*QM%gdN@ef zStg9_l5ST`S(bfcv%_|m%E|Y_9KbyQgeAaw6II=xBNpSEg0P4xCI;>X4%UmB#3 zP0PKk6h1l)X8y`tby|dyHDbt;;8DwCA7rubl5XO^n&p#ZoOGvK`BX@^2%Bf@F4l)f zcs>?buLwEd8hOL#nFNCPAa8t{yz$@|`a^y^|Br!Wajc*7xpA!F|NBp$9@gxCNBdtr z*#GY3^N0+n)=cJ#XhF>Uo0@c^InPp_Pe?(ND>`8+jvkTADOW_z3zOywwMH_@#K<_C zc|O@Al4Vr$Ylblf!F8Ia(Ib+x39=>fxR7kjZ&_;m_5Xb*CZG!vk>i`tBvLR*GM=+I zieFy5ztBRm=n**)vzf@r&nFioVrjm=fB4;m8RTLLe{K^s?#XM!S zXC9a0*C@VLiAY&5N;q7d2t6yZKPc7n`~Up(_~j3$@htuPI9BfegM+WWdRn*tA08e( z)PKI0&m(e}}B&$_F)=NSd2Bxi7U|tWQ8h1F=aJTTFw(am&^)bMDaXf08^2K=1w++=XgF*dt^%0l&L+E zai!HBQ4IeIsaa}&jJTfBLSZ|mi-O5aOfoT1aTGmzL@rG#m~`R81pbas*^m zLerF_G*1^PyT&36s^9Cm3oH`_lR#3Y^AU^!#(FFBI?t7+S?1sN0keQ5ie4@GN2Cxb&{|EieiZE*dM8rXb0E0X2)|@Tm1fcmu(j)#!aI2^3KOvuRAzMtbO^fz z3XS8NvV=-PuPM(!r`w5kiM|o08H^^=Cd?|M*Q^Dx+77b{QNxU!;OtVD^)eck7v{`AY*?68Q24fC@zW0UM{Nk?EYTF@+Ihty# zu<9uzuV9`n-y<`c(+QJiJq;woI*y8KreOX`@Z=!=`riyUd%-j%$AdFPtRfxllEKd~ zT&9jS43A-=&`7{I2b~~WKm&L_meg;QXxFsn`sm-V0Y4QJNmJKQJZ2C@1612c4tlbl zQdoIRGBSk`88HUqkk4l$CW#osXyXA%#b-sv#|sC`EjsA!3}ky(uA4j_3rR){m@*kt zo{4L_EKxfJC3DC1<0!r**Y?zd7zaBbCYMvDn8!}DOx&2RD+Z5>@t7x^X5_7`PCabF zR3ukSV!ZBqvlf6bfm)(Whf@K)cQrP#)Goj}e{p<5gsh!HAyQ;a#pGDwidV2#O&wl8 zxML}1z9Zj$a~SW(2SCcc{pKj%kN2y6+atqsk+Gru-=-}YqRc~vRouf;=Ok(>=2=Qc zjHE1MK+md1kZX+kld9u_J=E!Q!{(mhQ>)Cw@ghcXD zuc0YBdxlkPFeT*xfbNYF<hFsl)>w(mJ`FIf3wuE|2o zB|c*L%9)|(wS9OgJfZg!cCXCm60jNr3o%vrm0|EinEe@$D@XS1&UH-7b>5_NA*

    w$fop^aY@8YUV?Jm2FVP0Tj|8R$B!z%cl?q; zsRDanP)TP@1A9Yy0>VgjC8StX*7e+sP85Ahc6Tp%vAf$Rues7d)?DPQvJ7R269=e< zhJxq3ZSl=xg(F+~}= z_#Mlc1o^`QiJ}i=;C_)0_`SM-4ln<_^S z$p=zKjJw*|oS_a4ds~a3M^@5>l^he0x5s_I=c`p?Q;{g3j|nTZ>Q&Zkk1E9{dAAU0 zHwWeNk*l%YrFq(=x#nG8b^8Mmm<)=1Aj9834vFcAy$g;1@tR4=Q-+H|fUw{TfP+*G z+(aSiYwU$tA=C=#G5KJ>0lH4dp_}U|pmHmODZ8)n<>Q|&Uy;X_TNhL*@;@xCz@Q`O zx|v+HLq33Hm;vxH%@hmZf6uTtFWH#qc6DB{1(}L0tqL_h8iwb@#VpwhL$&4u*su%6 z;u)C!aqa_80E`m0S-BXfhOsZhXNdmC zCVVm#;;IU=QyT6&{mzx38J*F*hn5|TN9gQzG@W$%o$LMhU_ajPtY}2w1%URPmp%cz z#mjz`2+%oVWVza0G___0a!n_I^^#8RhR$`c5rr4pOvjuA=cet%dCLuW;Dv27*J#NH zsO}Q6Mtv)cP?Wza6s6R~;@WJYR6UUVnrCdnPE|q;r|R@^@9cGw;FMmc|9dBuqUiMB zc8*`a?!5cB*OBx5SiKSXxe&V3*K*G6wVxD|o&Lf8{{CJ^u>@GhK#DQX7=Y1>f_3_x z^LeiMj9EI{`RKass!xRk!I%NXIJmn9x{5sE#%!2fDSaEL+EO{I5R91A(Gv9kHX<1Jj zx_=6nLakm4@37)Ixv+Zl1(G!2;Q;aNyAa@6rAT)P6!^%eL4NJ2qz$Jun*fzs3jc~C zZgJq`PjrnJy?7h@l|n<#f>TUEeeAS$_=B!Rnz+rfyJ zl4E0+rc|cKx#q`#(DAn^LrJ4qrYKdlv5<+GTRG1`Z&3{Pjb`XEM;QvOMqCk!bSUF$ zYBs5rU99LBHo$BfyHnGbD9hXO`8=PTp<}>aKD}kh9Jd-(L==TkrZt5#X?{*A4v%=^ z?{;~vJ7Su4$Bd%kv=oPcI=k_H$o321I@GwtS%gtz=Gh33any2%6Jm9M8@<+^^6g$m+_ILa$z+8DhewC zg|Ic9RDpgSAI1k=niV`}mA3;2zX394lI015nFT#xNHP;>xi>)oMm#|lg_ar^l2Y@x z#<>RCiLs;=c`9HVYe@KJ;&ZF`RVUdP=OUak1VB8^maA!YYt^*K+gnWYGEe4n#p)Pzm8I9cJ$IfBy(Dc<~|=$rZ4sgr~C8?;JdP9v?n^67R?R zy@O|+k00NCe79~a-tNCED>%K~e}`j@c4Rb(F%ag0VSoANf@qWMu*NH!+u4lXie=;j zta4;v)#)r(O&~Lb{LD%xvDqaS-kwPa(Ded+&JxY9fk?t{E6_BnzU|epEY%DNaAm20 z!j6tDf1Yy+`8l9&g`44w%4 z);VW1{hQ>P{g@}L^R5-aYDxmLV=-;k;9&Wl=Em^0)aFKRyacIn9=2|e;=i%xk%K!` zT6?hn9q+UPGMpcURsiTs9!4$*z+2t%!aPs#g%&lbp?ndxRivc6PHbHxX$fjmuSNQU zccYc^!{xNb#-(vgExfs&o@5*p543n;z(FCD;;^)}aP|R` z;6vyZAPJM2k9k62bN~X&HT*(t8fbDS0NWv`n4tBglZ#Hv_?R=yDBGKkEyy*^cv>;w zc})%FV1(;*_$)))PN~PDkHbvM9{DTCTJubE7iodu6b?R;6rL;1XbQ5pZ5{e(*)5$F z8N-CSGm*|SwnH8(#>j|e;$|ltS-5I#>I*u{mb(?t=u?6~Tkx|Vv*3IO+ZtM$gH8+Om zT~HBtLREK0Z@Zb1t1{!zHe5^oXfFRNKT{2k{Dkz{q-<$r(#IDm_j&3PuFQh}uvrm}(pW zMXwF1t_K%nBB2?1EOPAF4m4JoC9Skw*iL}K?Ijs@oI4L6B{{n>y)o`OH2B@nh1IgJ zVdrYA;=|T3jLVH8vD0Gwla;PK-`^((wa%cPa4aXvON+{h{8cAG;Ej}%`OF$OmZXyq z?1}NQZ4TKv<<`RWgvEe`)oyN)a*j;MqB9d4QWBH1u^&6s&4Aq12^QQoCXW_QXtNa< zVrqM722e_BjVZP-DkQt1#PiqJKV7_f`J%IDzM+M=0-oj9f0bf}B88wSVfi(e!iZzn zRC0i*jN9Ma*L(h`dCLF>Mm}>y5x|#DyvpgtKz1*us1 zN|pYX0OmH0ZYkh6w;1>~(BT zui(Lt!`2=(h_h{ zC`jAfZ<~x(t>MNqgXN2y=3>KNReegbOJHF_h`m*30RjTo$3Ot9*wJu`3+9YWr-g>04v1)LP)ELNfgYfTU#EO z&E4#q`0VJb=M@mp6J8;Q=kCg%mL697)It!;Zmy+>HF%qoL|Gfk7MfUte;1`zYbQx-;6Z2!>%_zu*shce(L>T6fb~L39?2%oHqcVR}IgM zqq4VhBjdrqdExiqh57xb-GO0$8up*--rAj4fwTF(Qx4zciD3-SZX`2f@aJ8 zR>iRKZmXc&e6O|i=b#<-X8WuSZne>%p<6e_cakP-P#>zZ9JFTzLd1$EZR~@%Avcnv z(>tSbVk`$Lr-hmdtzz={>TBg=D7hj>`)cPdLIQAT_){t)y+`mfXk*tp?n@qAT_852 zB#z$ei!FG0T`kp_IxkHC?Lme*Q{BK;P?DED!pG{$u)z~Ghg_(Y?Jh&=kNvYKf{*W%=L%d`*B66m8fE$mcinGYfY0j`XgtVcW1 z&mHg77PI7xZ|J-6hgZU|a?6cMg5=*c=h2p9zF^#IIqN_sg_U51EK{PebmfVzo--dmWkdUa$z#)-Ax62sIQ{PI4LQF!CWG^{pN}t3 z$@iy!$2-5|aU9z}c;WQT%f<(X>$|?e*hBB3yKRF%S4{pd<00wy3aY2uPJ!j(A^R?! zP^M7<@AAw`ybnnc8bgC2MPs2DTjRWyXL5GUB$$SNNu0!Y2}EFK{lgTjBs7Svqx?)WYV6(K{rXtZUI;0 zKq(Ut$D}ij{G=G3{mrm)A@DT!#2#VsBqkjgwyRFOmX~R&u2hevGoEv3OxIdfe0&`1 zqQk4mSl1&#Q)g29In?kPCC(AQt+gZ&9UI)(W?0L%W>7V{5E!>lv-% zo@;@*_b$a1L&7c!O?cje^)AN5--l5>9-NUxvXtc@{-Xm)ghXYX;o3}VIfoz;`qzW& zE|HM!Vy}gLHkKU2>9zb=`5+uxKhwDsaMP@B)llSirC&~2C3d$4+(-xLT*-w-|1etn zj9>LS8X5G(4~_2Cx~XmW#8X zSAqKF+p&%ohdl+?%MjdQ53YBWd3qi6YJ6)r5@*#ew1&GLXE$`= z{l_6}5G<9J>ma+wwupRf#uf(GuE3X@k_7>-(OfM?;y~H$!^Y&UXwp)M z?m>L?B=&ynYBK*aY;{adj>Fg%bO%=f+5G+?paw9jVU+3m12AFE`w-MiJ|1({eZ{hD zMsxD`<*OZ(WZA6^BM(h{fa>w730biL{zHray{*w#%&(OoA)X}50fendZ4xO@?**EK&F{Os0FfW!tqwpV8ZkD|X|eUDOr3fsWZcA(>?( zjU1vKsJmm2NM5@3;&2Js-0rO~+)P>iQ?97yYK*RWGdySn3gX%J$dt+yBy=Ulx&a9u z=(VDm<2mt#Z6{VM82<=CZ1rUFP7bP{ zp;hIyl*ef4x|6iNjD&k@32hlZ_cMmhg?RzCiJqfN@pffAorSnf5M0^5O6X9dBv5@c z2nX8e>GmL>${1}E=d(1pEulWki{R5wNG`9Z)kh`QTM8=>?|MXXd9UuQ9f~buh%S~h zTCAG?Eh34Qb7ibo%I+Rb6e^Xztt~M(ZR%Cyd!H}5rUk(l39xCw^~GasTJV0pFq;+x zpDEI&fjMsFQn)i+XcHaF2DQhLl$m7e?x({bmc&E)$6UxchA#fVb3U8Tf&^Q}JVT;StB9T#<8?`zEmW~(ki-_SdOKQ|7F5C zn!$gbhz<{XeQZNJe`(-TD0w%QFH-JKSMF=aj zm8yCVJ-cx`@&{vlcU>*>>4gSv*s)E~!%I-_n7PSezvtXdTc>LB{atILFcN$>oG&!Q zZepjXCY$tJLR>R>Be`aIizM%mm2-I9yq<$e-grfq?}X7@FH1<3h^}xK2H3U;yYl%; zH_MK-y75b-GWBEeTF~?b%_tth>zQlKCdx8{s>v5soqKJws1TYjLUTwZ#f`VpRcT^% z!Yhkr^-QnJBvc`UTO<)$N&TC$zqrvfE}VhCw?6$#6>8N}RfT$fr$jIoelt4jeyZZR ztb3e};{EvRT2hz_oX?dU#-@eYEL@~5t^j~_5mMKNOLd(avDajH^F8^5%rg$gx(qbI zdRzmj*$C(_W3ondKi0llGUSk$xP@a6;Vxx-Cn}M&FzaX)NpYZ^3C0U@%PHhN#41yM zez0E`9zvilW~rX2x(aRC{R>0uGf;Oi*-Z-;gv16mJJm)Vpn6)x*0ayF;xO954w2ubK~%sU?F=_ zZOetQgx%y5F9qy@6a|w`@V;O=33eOZ4RI!YMx99Ralv3#4@Z#W9HK8e0!ip z8FpuKw$HGf#YS7D){QW-2AVt-H)M6dRql{pth0802NzQG9@NIlKX!DN+lhiD*?T5xEU2+Z7Hs;FI`D>WMFh4#BST!Hk za}>MBZ*Va^Db(x#z+=Qpy_Ch@bn3JnW{qLie2tX zW(O8AxYWfADu#5~K=c+df>E#X}yT!8s+1a21&|M~HgS7)z1$1dLHZCjP zu8oZknlK*nxDHh>3T9F^j96d@VY1CvTnA8+#Ckide6>;MvHBv4o&IX&d zE`WQ1%kvz|DZ_~M2MXebCl$4(AguMutOapBE<+Z`({!&|uI$;dqbtI>pkS_|DPmKDmSnlkxOfh?)% zT&LnDkJqr3TGeJJ8OWvD4fr6T6AGNcwQ-I1XT^2ny;$;EIAPx2ZS-3lE|~y4OpM`d z2WoFOLOnX#Z-;9Ju^q6x-w63hD_nCpq!p~Yy?J{LxwS&a6Gbn?mZ1klGHtGOmTdPf zrrra_b!K(RnLn&y!zW$jlZLfsf}Db|-q=e}Rg)36vjJ})p`DFM2M9#RsPs4{taSiO zG%Gid3Ap!?tP8YXGNCQgZFsdDH!B0x;zyzPalvK{(fegwcwl?*D)~J)p&hQ+x{$5s z7ryphxdim*wZxOBx2DpSAPJYre1>TZxl(gxsTew?)7)fE$xMj>ToEEk<`PS^!E&HH z4^!dQ0`oEX;#4SarSxsBE(gOe_DpfJhEa4C0lBYH*|Ho;__tb8pGYzAgUd-6?c*`D z)ZnnteAS{;_q@qhU+L^W=5zrXn#lMyll%M6Ow#-J4$9IPRO1;z-TRJdUr@lKs)aDx z=Qc3cu0v#b=2}EG!}j69YWSZax>bzBnW+b|wBY^Xk*+Ox?-YpI*u$ebqNOQXKNq2Q zq59ggH=)syP(rGB6l*KL*utggA=csvTUUHNgzyeaz717D@S{A#nt@-5b}kr7Nx zx(j77|Gl15V(uEg?B)^WIGd*1;l)50k=X*sc#P5jarS>q4L`{!^BwzdDudIa0B z{*zsSZOt;9&dz{0QcuaY-@qMd*koN@z{#}GkF|6d>dL+6JwT6vtJ*?&%P?_ zYL?2#`8;DmplzAwB2X^u%Jqk9fd$W)rYY65Wh2%QRMjGrrz7#CxoJfYssLK(RSiwJ z&0mQdBF38K)+=ZaG+_c^tUXp%DSJGoS*FN{CRdog!Pexn4{(vcF9CC(d}JrCS4vSW z!&})eeu-PTCFnd?M9%ZvRJe}Y>Z4R)OXmLlR6Sobe;e)d8!VY-jwNWNU zu4Md6kzhW4TSKql@hp7=*l`@H+b=R^)`d9$T;@qG)36M?U06t0Z1D{ms^9MUukVIF z_dm`?SaN+BV_6PcVq>o?#r-ej6AQar0IsI`FA*W8iMXM@F=};rRny5gPviagsg2je zTVz#5c$2ZtB{xC&W2p$^4cAkilc6O6!;+}RQFLq*K=EsqE%pc{e?R`=wT<=#K}mVj zHWmaj`Kuha2}1R4cI-lmYo20gAb_ZqL&s5R8|=zRb;p&{+btsylxY9Em}46Z!z-gw z-v(gbJq~{Cm@4N+vNPiaLwM1AQXNLSEMZ;-OwWYU);dx<(px*Ujwh&;GH2S&0c+z+ zDJX}*lwv+Hwb_S3BZp+3r%cw1JYZ(V`g!n}Trf-vccsjEcwf?!UQ?c-$Y>iBN6}lG zog>JE`R*|+YgpCdCeH*-ZLUKu?64~=``#;S*|Ys2 zxpCaS;$E!{jVh(~<-w%N)Y)ImGTc}piazN)+wYil^+ssckD`N^T!yVnajcSG`{`M|M>TikV<-E>Bn3#iT>nVBVDDr z=92wphOYNC-VbH~z*g$nXwk(m`#w2Xx@T+M^vTnx)|@BtEiNka>*&snds1l}g4`Cw zQDgs1Y=G}w**SZ|c8xr>BOXP5Bl~Ha7bwl7-cf0pH?9-Eb^l_X|Cap4u&6Hb_AX5H zx8yIloW32}7@sOB3xL47@1h-{JL6f#oyvW*zrWwUl9w_(Y_*tMaP|#Sv#`&`CYIYK zn+4e*XAIhawOAPU!S}5+4OK##6NauI$P_Sgcz<|5C!K}@hjeN1XQClJp>~!-Sm037qy4L8a!&Jfo6ulU>>i zLrTo(Z6fk~Xju(xM`KUGXC%#4K_y5;(>W$^nlKedmx82%D6xkF0}~6P35d6t!4!`$ zr5IR$QY|-Zke11?hHi0~`P(Bw9kX)j1~2p@;yf086z3yC!Ypb2qLp|0xbC~8H{y9u zP3^DlZDPND=;9Y*Gjx&e4aphM9LZuoINu!L_An2!WGZad&2Nd%x7IWB(wohC!Jxf| zL4z^t**2Rk7M!WHXH26AjVeA$Nd=WcttEY? zfd|%M?8MQrvBV^>trMhrQ(6=(#}lkmMubeJOlefkxFQA1eQu&$l#b1DS^ihW(K%OF zSj*Z}pd%xeP&882`+x`20NxOf*{JCop|RS^u{%F%meg>M&`eLEU%07!oA=7I03r>H zzb=nyYK+AhQ=?A>w&O zcrNR#acyXnjrTNpyf{j-H;#Noct<)>hb!H*1Cvb+hynvLW$oC@^y6mC%EqU#drSi<~7JRVulCc$q zmkGYrC$mM@Xb&b%rt4w(XA6;T1@}>D)UbT1a+?-Kzholr^ZV~z5+4@@nia{%kM{PX zN9YDqb{&B5R9H)wGrn0xFcUXBdu|!RESNX6D21CqgH;Z`0pIsOVL0JZpFBJI>UnUP z5LeB+PcBae4KEz7dBX_MW+mfxFyU23a0x8(;bDNVM>9+#M~0QN1fcR7;}oVow#Nig z!c&v)F3WIt(Y(Gjuh_znt&1>mtqe^ZVfxzCrH+lFpHLD6N^7LmY(cJaag&n;(>9dD zMGQBj#sxN-2niyjFl?!UQyTURyTuHM6In|;!CX%T+D5XBbb>Z^Oifoc6;kimLYX53 z3zTC@uQ?1lcgG2UTJn6-uVtIK;Tl+tQy+46LnQ0s`UfS)<_RCoHAw|CtlYeVJq!C1 zlOO1>7{Ly7N=QE8Fuf^P$y_NLT4cjbMj4xtDOHw{0}C|V>CwUAL2Rs2g%pKQEG0e| z2F7GArE!dK*l9q;oi)TotOSWuVJPK1hr1)Hrp|fi1x-IHBoh;!7;XJn6K7LZVg9;R ziIPnmjck9kpSHDKA@qxLB?*)kr%QpGhR27_HD`?2;0s zQrj08nK*iactH^}8VksHX8b^^tPU))V9e!gErBe#f|axJd{y#5$I;WW&jux|9iE$V zA1KrsH@?wj0j-s>RkdAO^2vlrBVO3B1pXUxzCOXT!!F-En(o>Z{GgD;!Y zXrxg$IL+*ihB%8XzS<2AIV`B56C=iQizSW8->4$HyJhXa-Cc4i$nI`qijUphvc$1b zVYxzX7e`%lmSm{`F0LFU-|f){5z;G=o3TGUi1$giKHL}NaWKU%Ss|H`BX^7>YfXf( z@WUBN%LkqQKe!l}sDk-Od8e|m7>zm0QdL$Ns2ohObmWF%DGrZ?=i%DBK8O#V_oHal zxv9AplG9A}9D<(BIns-wW3yuvDlC>v1iceE=tQc5(cDtRW;CDDoIoQ2Jrrx77zco> zgl+ckOk7j6i}=)^ZAn|=YFZR)D|c_LOZl?&_T#g0gW|puMcu7F(eYf1vTy*7myr>i z$I~jRZ>KDqbz%5R 0.2 + and + erpc:fallback_request_total:rate5m > 10 + for: 5m + labels: + severity: warning + annotations: + summary: "eRPC fallback traffic high" + description: "Fallback serves more than 20% of eRPC traffic and more than 10 RPS for 5m. Check main/router/processing health and fallback upstream quota." + + - alert: ErpcFallbackTrafficHighByProxy + expr: | + erpc:haproxy_fallback_request_share:ratio5m_by_proxy > 0.8 + and + erpc:haproxy_fallback_request_total:rate5m_by_proxy > 25 + for: 15m + labels: + severity: warning + annotations: + summary: "eRPC fallback traffic high for {{`{{ $labels.proxy }}`}}" + description: "{{`{{ $labels.proxy }}`}} sends more than 80% of its eRPC traffic and more than 25 RPS to fallback for 15m. Check the matching main/router/processing pool and fallback upstream quota." + + - alert: ErpcFallbackActionableFailuresHigh + expr: | + erpc:fallback_actionable_failure_ratio:ratio5m > 0.05 + and + erpc:fallback_request_total:rate5m > 10 + for: 10m + labels: + severity: warning + annotations: + summary: "eRPC fallback actionable failures high" + description: "Fallback actionable failure ratio is above 5% while fallback serves more than 10 RPS for 10m, excluding ErrUpstreamsExhausted. Check fallback upstream health and quotas." + + - name: erpc.latency.rules + interval: 1m + rules: + - record: erpc:upstream_request_duration_seconds:p50:by_network + expr: | + histogram_quantile(0.5, + sum by (network, le) (erpc:upstream_request_duration_seconds_bucket:rate5m) + ) + + - record: erpc:upstream_request_duration_seconds:p90:by_network + expr: | + histogram_quantile(0.9, + sum by (network, le) (erpc:upstream_request_duration_seconds_bucket:rate5m) + ) + + - record: erpc:upstream_request_duration_seconds:p50:by_vendor + expr: | + histogram_quantile(0.5, + sum by (vendor, le) ( + label_replace( + erpc:upstream_request_duration_seconds_bucket:rate5m, + "vendor", "$1", "upstream", "([^-]+)-.*" + ) + ) + ) + + - record: erpc:upstream_request_duration_seconds:p90:by_vendor + expr: | + histogram_quantile(0.9, + sum by (vendor, le) ( + label_replace( + erpc:upstream_request_duration_seconds_bucket:rate5m, + "vendor", "$1", "upstream", "([^-]+)-.*" + ) + ) + ) + + - record: erpc:upstream_request_duration_seconds:p50 + expr: | + histogram_quantile(0.5, + sum by (le) (erpc:upstream_request_duration_seconds_bucket:rate5m) + ) + + - record: erpc:upstream_request_duration_seconds:p90 + expr: | + histogram_quantile(0.9, + sum by (le) (erpc:upstream_request_duration_seconds_bucket:rate5m) + ) + + - record: erpc:network_request_duration_seconds:p50:by_network + expr: | + histogram_quantile(0.5, + sum by (network, le) (erpc:network_request_duration_seconds_bucket:rate5m) + ) + + - record: erpc:network_request_duration_seconds:p90:by_network + expr: | + histogram_quantile(0.9, + sum by (network, le) (erpc:network_request_duration_seconds_bucket:rate5m) + ) + + - record: erpc:cache_get_miss_duration_seconds:p50 + expr: | + histogram_quantile(0.5, + sum by (le) (erpc:cache_get_success_miss_duration_seconds_bucket:rate5m) + ) + + - record: erpc:cache_get_miss_duration_seconds:p90 + expr: | + histogram_quantile(0.9, + sum by (le) (erpc:cache_get_success_miss_duration_seconds_bucket:rate5m) + ) + + - name: erpc.daily.rules + interval: 5m + rules: + - record: erpc:requests:rate5m + expr: sum(erpc:network_request_received_total:rate5m) + + - record: erpc:upstream_ratio:rate5m + expr: | + sum(erpc:upstream_request_total:rate5m) + / + sum(erpc:network_request_received_total:rate5m) + + - record: erpc:cache_hit_rate:rate5m + expr: | + sum(erpc:cache_get_success_hit_total:rate5m) + / + ( + sum(erpc:cache_get_success_hit_total:rate5m) + + + sum(erpc:cache_get_success_miss_total:rate5m) + ) + + - record: erpc:error_rate:rate5m + expr: | + sum(erpc:upstream_request_errors_total:rate5m) + / + sum(erpc:upstream_request_total:rate5m) diff --git a/helm/environments/prd/erpc/templates/rbac-vault-redis.yaml b/helm/environments/prd/erpc/templates/rbac-vault-redis.yaml new file mode 100644 index 000000000..e55bcebc4 --- /dev/null +++ b/helm/environments/prd/erpc/templates/rbac-vault-redis.yaml @@ -0,0 +1,44 @@ +{{- if index .Values "redis-ha" "enabled" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-redis-vault-secret + labels: + app.kubernetes.io/name: {{ .Release.Name }}-redis-vault + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation + argocd.argoproj.io/sync-wave: "-2" +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + - apiGroups: [""] + resources: ["secrets"] + resourceNames: + - {{ index .Values "redis-ha" "existingSecret" | quote }} + verbs: ["get", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-redis-vault-secret + labels: + app.kubernetes.io/name: {{ .Release.Name }}-redis-vault + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation + argocd.argoproj.io/sync-wave: "-2" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-redis-vault-secret +subjects: + - kind: ServiceAccount + name: erpc + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/helm/environments/prd/erpc/templates/serviceaccount-vault.yaml b/helm/environments/prd/erpc/templates/serviceaccount-vault.yaml new file mode 100644 index 000000000..93158905c --- /dev/null +++ b/helm/environments/prd/erpc/templates/serviceaccount-vault.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: erpc + labels: + app.kubernetes.io/name: erpc-vault + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + argocd.argoproj.io/sync-wave: "-4" + {{- with .Values.erpc.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: false diff --git a/helm/environments/prd/erpc/values.yaml b/helm/environments/prd/erpc/values.yaml new file mode 100644 index 000000000..3cc75d4ef --- /dev/null +++ b/helm/environments/prd/erpc/values.yaml @@ -0,0 +1,1972 @@ +# ============================================================================= +# HAProxy Smart Load Balancer for eRPC +# ============================================================================= +# Provides intelligent traffic distribution based on real-time pod metrics: +# - Goroutines: Direct measure of concurrent work (weight: 30%) +# - P99 Latency: Response time degradation detection (weight: 35%) +# - Request Rate: Actual load distribution (weight: 15%) +# - CPU utilization: Resource pressure indicator (weight: 20%) +# +# Traffic flow: Client -> erpc-haproxy:4000 -> erpc pods (via headless service) +# Auto fallback: erpc-fallback servers stay at weight 0 until Prometheus shows +# sustained degradation on any public eRPC pool, then the matching HAProxy ramps +# limited traffic to the upstream-image hot standby pool. +# ============================================================================= +# Custom HPA for erpc-haproxy (overrides upstream chart HPA which uses metrics-server) +erpc-haproxy-hpa: + enabled: true + minReplicas: 3 + maxReplicas: 100 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + +erpc-processing-haproxy-hpa: + enabled: true + minReplicas: 3 + maxReplicas: 100 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + +erpc-router-haproxy-hpa: + enabled: true + minReplicas: 3 + maxReplicas: 100 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + +erpc-haproxy: + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/haproxytech/haproxy-alpine + + rbac: + create: true + + serviceMonitor: + enabled: true + extraLabels: + release: prometheus + endpoints: + - port: prometheus + path: /metrics + scheme: http + interval: 15s + + # HAProxy configuration for HTTP RPC load balancing + config: | + global + stats socket /var/run/haproxy/admin.sock mode 660 level admin expose-fd listeners + stats timeout 30s + maxconn 500000 + spread-checks 5 + log stdout format raw local0 info + # Raise file descriptor limit to hard limit (was soft=65536, hard=1048576) + ulimit-n 1048576 + + # Performance tuning - maximized for throughput + # tune.bufsize: larger buffer for big JSON-RPC payloads (eth_call responses can be huge) + tune.bufsize 131072 + # tune.maxrewrite: space for header rewriting + tune.maxrewrite 8192 + # tune.http.maxhdr: max headers + tune.http.maxhdr 128 + # tune.recv_enough: receive enough data before processing + tune.recv_enough 16384 + # tune.runqueue-depth: scheduler queue depth for better throughput + tune.runqueue-depth 512 + + defaults + mode http + log global + option httplog + option dontlognull + option http-server-close + option forwardfor except 127.0.0.0/8 + + # Timeouts tuned for blockchain RPC (some calls can be slow) + timeout connect 5s + timeout client 60s + timeout server 60s + timeout http-request 30s + timeout http-keep-alive 30s + # Queue timeout: how long to wait for a server slot + timeout queue 120s + timeout check 15s + + retries 3 + option redispatch + retry-on conn-failure empty-response response-timeout 503 + + # Main frontend - accepts client connections + frontend erpc-frontend + bind *:4000 + # Frontend maxconn: match global maxconn (no restriction) + maxconn 500000 + + # Add request ID for tracing + unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid + unique-id-header X-Request-ID + + # If every main eRPC server is down, bypass the metric debounce and + # route straight to fallback. The controller still handles gradual + # degraded-performance activation through erpc-backend weights. + use_backend erpc-fallback-backend if { nbsrv(erpc-main-health-backend) eq 0 } + + # Route health through the serving backend so fallback only counts + # after activation or total main outage. + default_backend erpc-backend + + # Main-only health backend for instant total-outage detection. + backend erpc-main-health-backend + balance roundrobin + option httpchk GET /healthcheck + http-check expect status 200 + default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 check + server-template erpc-main-health 100 erpc-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none + + # Main backend - smart load balancing with dynamic weights + backend erpc-backend + # Weighted round-robin caps fallback traffic by configured weight. + # Controller adjusts weights based on Goroutines/Latency/RequestRate/CPU + balance roundrobin + + # HTTP health check on application endpoint + option httpchk GET /healthcheck + http-check expect status 200 + + # Connection reuse for efficiency + http-reuse safe + + # Aggressive health monitoring + # inter=2s: check every 2 seconds + # fastinter=1s: check every 1s when transitioning + # rise=2: 2 successes to mark healthy + # fall=3: 3 failures to mark unhealthy + # maxconn=5000: max connections per backend server (100 pods × 5000 = 500K total) + default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check + + # DNS-based service discovery via headless service + server-template erpc 100 erpc-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none + # Weighted fallback is intentionally capped below fallback HPA max so + # partial fallback traffic cannot grow with scaled-out fallback pods. + server-template erpc-fallback 3 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 weight 0 check resolvers k8s init-addr none + + # Direct fallback backend for sudden total main outage. + backend erpc-fallback-backend + balance leastconn + option httpchk GET /healthcheck + http-check expect status 200 + http-reuse safe + default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check + server-template erpc-fallback-direct 40 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none + + # Kubernetes DNS resolver + resolvers k8s + nameserver dns kube-dns.kube-system.svc.cluster.local:53 + accepted_payload_size 8192 + hold valid 5s + hold nx 5s + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + + # Stats/metrics frontend + frontend stats + mode http + bind *:8404 + http-request use-service prometheus-exporter if { path /metrics } + stats enable + stats uri /stats + stats refresh 10s + stats show-legends + stats show-node + + # Weight controller sidecar - dynamically adjusts backend weights + # Uses 4 metrics: Goroutines, P99 Latency, Request Rate, CPU + sidecarContainers: + - name: weight-controller + image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/haproxy-controller-erpc:v1.2.2 + imagePullPolicy: IfNotPresent + env: + - name: PROMETHEUS_URL + value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090" + # Goroutine query - direct count per pod + - name: GOROUTINE_QUERY + value: 'erpc:go_goroutines:by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}' + # P99 Latency query - in milliseconds + - name: LATENCY_QUERY + value: 'histogram_quantile(0.99, erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}) * 1000' + # Request rate query - requests per second + - name: REQUEST_RATE_QUERY + value: 'erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}' + # CPU query - percentage of limit + - name: CPU_QUERY + value: '100 * erpc:container_cpu_usage:limit_ratio1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}' + # Target thresholds - traffic shifts when exceeded + - name: TARGET_GOROUTINES + value: "400" + - name: TARGET_LATENCY_MS + value: "500" + - name: TARGET_REQUEST_RATE + value: "50" + - name: TARGET_CPU + value: "60" + # Metric weights (sum to 1.0) + # Latency has highest weight as it directly measures user impact + - name: WEIGHT_GOROUTINES + value: "0.30" + - name: WEIGHT_LATENCY + value: "0.35" + - name: WEIGHT_REQUEST_RATE + value: "0.15" + - name: WEIGHT_CPU + value: "0.20" + # Backend name must match HAProxy config + - name: BACKEND_NAME + value: "erpc-backend" + # Pod namespace for Kubernetes API lookups + - name: POD_NAMESPACE + value: "morpho-prd" + # Fast updates for responsive load balancing (matches pod scrape) + - name: UPDATE_INTERVAL + value: "5s" + # Weight range + - name: MIN_WEIGHT + value: "10" + - name: MAX_WEIGHT + value: "50" + - name: HAPROXY_SOCKET + value: "/var/run/haproxy/admin.sock" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "256Mi" + livenessProbe: + exec: + command: + - /bin/sh + - -c + - pgrep -f "controller.sh" > /dev/null + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - test -S "$HAPROXY_SOCKET" + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + volumeMounts: + - name: haproxy-socket + mountPath: /var/run/haproxy + - name: controller-tmp + mountPath: /tmp + - &erpcFallbackController + name: fallback-controller + image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/haproxy-controller-erpc:v1.2.2 + imagePullPolicy: IfNotPresent + env: + - name: PROMETHEUS_URL + value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090" + - name: BACKEND_NAME + value: "erpc-backend" + - name: MAIN_HEALTH_BACKEND_NAME + value: "erpc-main-health-backend" + - name: MAIN_HEALTH_SERVER_PREFIX + value: "erpc-main-health" + - name: FALLBACK_SERVER_PREFIX + value: "erpc-fallback" + - name: HAPROXY_SOCKET + value: "/var/run/haproxy/admin.sock" + - name: UPDATE_INTERVAL_SECONDS + value: "30" + - name: ENABLE_AFTER_FAILURES + value: "6" + - name: RECOVER_AFTER_SUCCESSES + value: "20" + - name: DEGRADED_P99_MS + value: "25000" + - name: RECOVERED_P99_MS + value: "10000" + - name: DEGRADED_ERROR_RATE_PERCENT + value: "1" + - name: RECOVERED_ERROR_RATE_PERCENT + value: "0.5" + - name: FALLBACK_ACTIVE_WEIGHT + value: "1" + - name: FALLBACK_IDLE_WEIGHT + value: "0" + - name: MIN_TRAFFIC_RPS + value: "10" + - name: REQUEST_RATE_QUERY + value: 'sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"})' + - name: LATENCY_QUERY + value: 'histogram_quantile(0.99, sum by (le) (erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"})) * 1000' + - name: ERROR_RATE_QUERY + value: '100 * (sum(erpc:network_failed_actionable_request_total:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}) or on() vector(0)) / clamp_min(sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}), 1)' + command: + - /bin/sh + - -ec + args: + - | + log() { + echo "$(date '+%Y-%m-%d %H:%M:%S') [$(hostname)] fallback-controller - $*" + } + + url_encode() { + echo "$1" | sed \ + -e 's/%/%25/g' \ + -e 's/ /%20/g' \ + -e 's/{/%7B/g' \ + -e 's/}/%7D/g' \ + -e 's/\[/%5B/g' \ + -e 's/\]/%5D/g' \ + -e 's/=/%3D/g' \ + -e 's/,/%2C/g' \ + -e 's/>/%3E/g' \ + -e 's//dev/null || true)" + + if [ -z "$response" ] || echo "$response" | grep -q '"status":"error"'; then + log "Prometheus query failed: $query" + return 1 + fi + + value="$(echo "$response" | sed -n 's/.*"value":\[[^,]*,"\([^"]*\)".*/\1/p' | head -1)" + case "$value" in + ""|NaN|nan|Inf|+Inf|-Inf) + log "Prometheus query returned no usable value: $query" + return 1 + ;; + esac + echo "$value" + } + + fallback_servers() { + echo "show servers state ${BACKEND_NAME}" | socat stdio "unix-connect:${HAPROXY_SOCKET}" 2>/dev/null | awk -v backend="$BACKEND_NAME" -v prefix="$FALLBACK_SERVER_PREFIX" ' + $2 == backend && index($4, prefix) == 1 && $5 != "-" { print $4 } + ' + } + + set_fallback_weight() { + weight="$1" + servers="$(fallback_servers || true)" + if [ -z "$servers" ]; then + log "No fallback servers currently resolved" + return 1 + fi + + for server in $servers; do + command="set server ${BACKEND_NAME}/${server} weight ${weight}" + if echo "$command" | socat stdio "unix-connect:${HAPROXY_SOCKET}" >/dev/null 2>&1; then + log "Set ${BACKEND_NAME}/${server} weight=${weight}" + else + log "Failed to set ${BACKEND_NAME}/${server} weight=${weight}" + fi + done + } + + is_gt() { + awk -v a="$1" -v b="$2" 'BEGIN { exit !(a > b) }' + } + + is_lt() { + awk -v a="$1" -v b="$2" 'BEGIN { exit !(a < b) }' + } + + fallback_has_active_weight() { + echo "show stat" | socat stdio "unix-connect:${HAPROXY_SOCKET}" 2>/dev/null | awk -F, -v backend="$BACKEND_NAME" -v prefix="$FALLBACK_SERVER_PREFIX" -v idle="$FALLBACK_IDLE_WEIGHT" ' + $1 == backend && index($2, prefix) == 1 && ($19 + 0) > (idle + 0) { found=1 } + END { exit !found } + ' + } + + main_health_counts() { + stats="$(echo "show stat" | socat stdio "unix-connect:${HAPROXY_SOCKET}" 2>/dev/null)" || return 1 + echo "$stats" | awk -F, -v backend="$MAIN_HEALTH_BACKEND_NAME" -v prefix="$MAIN_HEALTH_SERVER_PREFIX" ' + $1 == backend && index($2, prefix) == 1 { + if ($18 !~ /^MAINT/) resolved++ + if ($18 == "UP") up++ + } + END { print (up + 0) ":" (resolved + 0) } + ' + } + + keep_fallback_active_for_direct_outage() { + main_health_counts="$(main_health_counts || echo "")" + main_healthy_count="${main_health_counts%%:*}" + main_resolved_count="${main_health_counts##*:}" + if [ -z "$main_health_counts" ] || [ "$main_resolved_count" = "0" ]; then + direct_outage_count=0 + if [ "$main_seen_healthy" -eq 1 ]; then + direct_outage_seen=1 + log "No resolved main eRPC health servers; direct fallback handles outage until main DNS/health recovers" + if [ "$active" -eq 1 ]; then + set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true + fi + return 0 + fi + direct_outage_seen=0 + log "No resolved main eRPC health servers yet; skipping direct-outage fallback activation" + return 1 + fi + if [ "$main_healthy_count" = "0" ]; then + direct_outage_count=$((direct_outage_count + 1)) + if [ "$main_seen_healthy" -eq 1 ]; then + direct_outage_seen=1 + fi + if [ "$active" -eq 1 ]; then + log "No healthy main eRPC servers; preserving weighted fallback recovery handoff" + set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true + return 0 + fi + if [ "$direct_outage_count" -lt "$ENABLE_AFTER_FAILURES" ]; then + log "No healthy main eRPC servers; direct fallback handles outage while weighted recovery handoff waits direct_outage_count=${direct_outage_count}/${ENABLE_AFTER_FAILURES}" + return 0 + fi + missing_metrics_count=0 + degraded_count=0 + recovered_count=0 + log "No healthy main eRPC servers for ${direct_outage_count} intervals; keeping direct-outage fallback recovery handoff active" + set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true + active=1 + return 0 + fi + main_seen_healthy=1 + if [ "$direct_outage_seen" -eq 1 ]; then + direct_outage_count=0 + direct_outage_seen=0 + missing_metrics_count=0 + degraded_count=0 + recovered_count=0 + log "Main eRPC servers recovered after total outage; enabling weighted fallback recovery handoff" + set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" && active=1 + return 0 + fi + direct_outage_count=0 + return 1 + } + + sync_startup_state() { + if keep_fallback_active_for_direct_outage; then + return + fi + + startup_request_rate="$(query_prometheus_value "$REQUEST_RATE_QUERY" || echo "")" + + if [ -n "$startup_request_rate" ] && ! is_gt "$startup_request_rate" "$MIN_TRAFFIC_RPS"; then + if fallback_has_active_weight; then + active=1 + log "Main eRPC traffic below gate at startup; preserving active fallback weights" + else + log "Main eRPC traffic below gate at startup; preserving idle fallback weights" + fi + return + fi + + startup_p99_ms="$(query_prometheus_value "$LATENCY_QUERY" || echo "")" + startup_error_rate="$(query_prometheus_value "$ERROR_RATE_QUERY" || echo "")" + + if [ -z "$startup_request_rate" ] || [ -z "$startup_p99_ms" ] || [ -z "$startup_error_rate" ]; then + if fallback_has_active_weight; then + active=1 + log "Metrics unavailable at startup; preserving active fallback weights" + else + log "Metrics unavailable at startup; preserving idle fallback weights" + fi + return + fi + + if is_gt "$startup_p99_ms" "$DEGRADED_P99_MS" || is_gt "$startup_error_rate" "$DEGRADED_ERROR_RATE_PERCENT"; then + log "Main eRPC already degraded at startup; enabling fallback weight=${FALLBACK_ACTIVE_WEIGHT}" + set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" && active=1 + elif is_lt "$startup_p99_ms" "$RECOVERED_P99_MS" && is_lt "$startup_error_rate" "$RECOVERED_ERROR_RATE_PERCENT"; then + log "Main eRPC healthy at startup; disabling fallback weight=${FALLBACK_IDLE_WEIGHT}" + set_fallback_weight "${FALLBACK_IDLE_WEIGHT}" && active=0 + elif fallback_has_active_weight; then + active=1 + log "Startup metrics between thresholds; preserving active fallback weights" + else + log "Startup metrics between thresholds; preserving idle fallback weights" + fi + } + + active=0 + degraded_count=0 + recovered_count=0 + missing_metrics_count=0 + direct_outage_count=0 + direct_outage_seen=0 + main_seen_healthy=0 + + log "Starting auto fallback controller: traffic gate>${MIN_TRAFFIC_RPS}rps; degrade p99>${DEGRADED_P99_MS}ms or failures>${DEGRADED_ERROR_RATE_PERCENT}% for ${ENABLE_AFTER_FAILURES} intervals; recover p99<${RECOVERED_P99_MS}ms and failures<${RECOVERED_ERROR_RATE_PERCENT}% for ${RECOVER_AFTER_SUCCESSES} intervals" + date +%s > /tmp/fallback-controller.heartbeat + sync_startup_state || true + + while true; do + date +%s > /tmp/fallback-controller.heartbeat + if keep_fallback_active_for_direct_outage; then + sleep "${UPDATE_INTERVAL_SECONDS}" + continue + fi + + request_rate="$(query_prometheus_value "$REQUEST_RATE_QUERY" || echo "")" + + if [ -n "$request_rate" ] && ! is_gt "$request_rate" "$MIN_TRAFFIC_RPS"; then + missing_metrics_count=0 + degraded_count=0 + recovered_count=0 + log "Traffic below gate; holding fallback state active=${active} request_rate=${request_rate}" + if [ "$active" -eq 1 ]; then + set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true + else + set_fallback_weight "${FALLBACK_IDLE_WEIGHT}" || true + fi + sleep "${UPDATE_INTERVAL_SECONDS}" + continue + fi + + p99_ms="$(query_prometheus_value "$LATENCY_QUERY" || echo "")" + error_rate="$(query_prometheus_value "$ERROR_RATE_QUERY" || echo "")" + + if [ -z "$request_rate" ] || [ -z "$p99_ms" ] || [ -z "$error_rate" ]; then + missing_metrics_count=$((missing_metrics_count + 1)) + degraded_count=0 + recovered_count=0 + log "Metrics unavailable; active=${active} missing_metrics_count=${missing_metrics_count}" + if [ "$active" -eq 0 ] && [ "$missing_metrics_count" -ge "$ENABLE_AFTER_FAILURES" ]; then + log "Main eRPC metrics unavailable; enabling fallback weight=${FALLBACK_ACTIVE_WEIGHT}" + set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" && active=1 + elif [ "$active" -eq 1 ]; then + set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true + fi + sleep "${UPDATE_INTERVAL_SECONDS}" + continue + fi + + missing_metrics_count=0 + degraded=0 + recovered=0 + if is_gt "$p99_ms" "$DEGRADED_P99_MS" || is_gt "$error_rate" "$DEGRADED_ERROR_RATE_PERCENT"; then + degraded=1 + fi + if is_lt "$p99_ms" "$RECOVERED_P99_MS" && is_lt "$error_rate" "$RECOVERED_ERROR_RATE_PERCENT"; then + recovered=1 + fi + + if [ "$degraded" -eq 1 ]; then + degraded_count=$((degraded_count + 1)) + recovered_count=0 + elif [ "$recovered" -eq 1 ]; then + recovered_count=$((recovered_count + 1)) + degraded_count=0 + else + degraded_count=0 + recovered_count=0 + fi + + log "request_rate=${request_rate} p99_ms=${p99_ms} failure_rate_percent=${error_rate} active=${active} degraded_count=${degraded_count} recovered_count=${recovered_count}" + + if [ "$active" -eq 0 ] && [ "$degraded_count" -ge "$ENABLE_AFTER_FAILURES" ]; then + log "Main eRPC degraded; enabling fallback weight=${FALLBACK_ACTIVE_WEIGHT}" + set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" && active=1 + elif [ "$active" -eq 1 ] && [ "$recovered_count" -ge "$RECOVER_AFTER_SUCCESSES" ]; then + log "Main eRPC recovered; disabling fallback weight=${FALLBACK_IDLE_WEIGHT}" + set_fallback_weight "${FALLBACK_IDLE_WEIGHT}" && active=0 + elif [ "$active" -eq 1 ]; then + set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true + else + set_fallback_weight "${FALLBACK_IDLE_WEIGHT}" || true + fi + + sleep "${UPDATE_INTERVAL_SECONDS}" + done + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "200m" + memory: "128Mi" + readinessProbe: + exec: + command: + - /bin/sh + - -c + - test -S "$HAPROXY_SOCKET" + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + livenessProbe: + exec: + command: + - /bin/sh + - -c + - test -f /tmp/fallback-controller.heartbeat && [ $(( $(date +%s) - $(cat /tmp/fallback-controller.heartbeat) )) -lt 180 ] + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 3 + volumeMounts: + - name: haproxy-socket + mountPath: /var/run/haproxy + - name: controller-tmp + mountPath: /tmp + + extraVolumes: + - name: haproxy-socket + emptyDir: {} + - name: controller-tmp + emptyDir: {} + + extraVolumeMounts: + - name: haproxy-socket + mountPath: /var/run/haproxy + + containerPorts: + http: 4000 + prometheus: 8404 + + service: + type: ClusterIP + + replicaCount: 3 + + resources: + requests: + cpu: "1" + memory: "512Mi" + limits: + cpu: "4" + memory: "2Gi" + + # Disable upstream chart's HPA (uses metrics-server which isn't installed) + # Custom HPA is created via erpc-haproxy-hpa values above + autoscaling: + enabled: false + + # Enable the HAProxy chart's actual PDB key. The previous lower-case + # podDisruptionBudget block was ignored by upstream chart 1.26.1. + PodDisruptionBudget: + enable: true + minAvailable: 2 + + # Pin the public CloudFront edge proxy to the on-demand node pool. Spot + # interruption of a single node previously removed all public HAProxy + # replicas at once and caused rpc.morpho.dev CloudFront 504s. + nodeSelector: + purpose: on-demand + tolerations: + - key: dedicated + operator: Equal + value: on-demand + effect: NoSchedule + + # Keep the public CloudFront edge proxy off voluntary Karpenter disruption + # paths; this protects rpc.morpho.dev while still allowing HPA rollouts. + podAnnotations: + karpenter.sh/do-not-disrupt: "true" + + podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/instance: erpc + app.kubernetes.io/name: erpc-haproxy + topologyKey: kubernetes.io/hostname + + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/instance: erpc + app.kubernetes.io/name: erpc-haproxy + +erpc-processing-haproxy: + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/haproxytech/haproxy-alpine + + rbac: + create: true + + serviceMonitor: + enabled: true + extraLabels: + release: prometheus + endpoints: + - port: prometheus + path: /metrics + scheme: http + + config: | + global + stats socket /var/run/haproxy/admin.sock mode 660 level admin expose-fd listeners + stats timeout 30s + maxconn 500000 + spread-checks 5 + log stdout format raw local0 info + ulimit-n 1048576 + tune.bufsize 131072 + tune.maxrewrite 8192 + tune.http.maxhdr 128 + tune.recv_enough 16384 + tune.runqueue-depth 512 + + defaults + mode http + log global + option httplog + option dontlognull + option http-server-close + option forwardfor except 127.0.0.0/8 + timeout connect 5s + timeout client 60s + timeout server 60s + timeout http-request 30s + timeout http-keep-alive 30s + timeout queue 120s + timeout check 15s + retries 3 + option redispatch + retry-on conn-failure empty-response response-timeout 503 + + frontend erpc-processing-frontend + bind *:4000 + maxconn 500000 + unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid + unique-id-header X-Request-ID + + # Direct fallback handles a total processing outage. The default backend + # also has metrics-driven weighted fallback for processing-side eRPC degradation. + use_backend erpc-processing-fallback-backend if { nbsrv(erpc-processing-main-health-backend) eq 0 } + + default_backend erpc-processing-backend + + backend erpc-processing-main-health-backend + balance roundrobin + option httpchk GET /healthcheck + http-check expect status 200 + default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 check + server-template erpc-processing-main-health 100 erpc-processing-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none + + backend erpc-processing-backend + # Weighted round-robin keeps processing pods balanced by the existing + # weight-controller and lets fallback stay idle at weight 0 until the + # processing-specific fallback-controller activates it. + balance roundrobin + option httpchk GET /healthcheck + http-check expect status 200 + http-reuse safe + default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check + server-template erpc-processing 100 erpc-processing-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none + # Weighted fallback is intentionally capped below fallback HPA max so + # partial fallback traffic cannot grow with scaled-out fallback pods. + server-template erpc-fallback 3 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 weight 0 check resolvers k8s init-addr none + + backend erpc-processing-fallback-backend + balance leastconn + option httpchk GET /healthcheck + http-check expect status 200 + http-reuse safe + default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check + server-template erpc-fallback-direct 40 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none + + resolvers k8s + nameserver dns kube-dns.kube-system.svc.cluster.local:53 + accepted_payload_size 8192 + hold valid 5s + hold nx 5s + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + + frontend stats + mode http + bind *:8404 + http-request use-service prometheus-exporter if { path /metrics } + stats enable + stats uri /stats + stats refresh 10s + stats show-legends + stats show-node + + sidecarContainers: + - name: weight-controller + image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/haproxy-controller-erpc:v1.2.2 + imagePullPolicy: IfNotPresent + env: + - name: PROMETHEUS_URL + value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090" + - name: GOROUTINE_QUERY + value: 'erpc:go_goroutines:by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}' + - name: LATENCY_QUERY + value: 'histogram_quantile(0.99, erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}) * 1000' + - name: REQUEST_RATE_QUERY + value: 'erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}' + - name: CPU_QUERY + value: '100 * erpc:container_cpu_usage:limit_ratio1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}' + - name: TARGET_GOROUTINES + value: "400" + - name: TARGET_LATENCY_MS + value: "500" + - name: TARGET_REQUEST_RATE + value: "50" + - name: TARGET_CPU + value: "60" + - name: WEIGHT_GOROUTINES + value: "0.30" + - name: WEIGHT_LATENCY + value: "0.35" + - name: WEIGHT_REQUEST_RATE + value: "0.15" + - name: WEIGHT_CPU + value: "0.20" + - name: BACKEND_NAME + value: "erpc-processing-backend" + - name: POD_NAMESPACE + value: "morpho-prd" + - name: UPDATE_INTERVAL + value: "5s" + - name: MIN_WEIGHT + value: "10" + - name: MAX_WEIGHT + value: "50" + - name: HAPROXY_SOCKET + value: "/var/run/haproxy/admin.sock" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "256Mi" + livenessProbe: + exec: + command: + - /bin/sh + - -c + - pgrep -f "controller.sh" > /dev/null + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - test -S "$HAPROXY_SOCKET" + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + volumeMounts: + - name: haproxy-socket + mountPath: /var/run/haproxy + - name: controller-tmp + mountPath: /tmp + - <<: *erpcFallbackController + env: + - name: PROMETHEUS_URL + value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090" + - name: BACKEND_NAME + value: "erpc-processing-backend" + - name: MAIN_HEALTH_BACKEND_NAME + value: "erpc-processing-main-health-backend" + - name: MAIN_HEALTH_SERVER_PREFIX + value: "erpc-processing-main-health" + - name: FALLBACK_SERVER_PREFIX + value: "erpc-fallback" + - name: HAPROXY_SOCKET + value: "/var/run/haproxy/admin.sock" + - name: UPDATE_INTERVAL_SECONDS + value: "30" + - name: ENABLE_AFTER_FAILURES + value: "6" + - name: RECOVER_AFTER_SUCCESSES + value: "20" + - name: DEGRADED_P99_MS + value: "25000" + - name: RECOVERED_P99_MS + value: "5000" + - name: DEGRADED_ERROR_RATE_PERCENT + value: "2" + - name: RECOVERED_ERROR_RATE_PERCENT + value: "0.5" + - name: FALLBACK_ACTIVE_WEIGHT + value: "1" + - name: FALLBACK_IDLE_WEIGHT + value: "0" + - name: MIN_TRAFFIC_RPS + value: "50" + - name: REQUEST_RATE_QUERY + value: 'sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"})' + - name: LATENCY_QUERY + value: 'histogram_quantile(0.99, sum by (le) (erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"})) * 1000' + - name: ERROR_RATE_QUERY + value: '100 * (sum(erpc:network_failed_actionable_request_total:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}) or on() vector(0)) / clamp_min(sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}), 1)' + extraVolumes: + - name: haproxy-socket + emptyDir: {} + - name: controller-tmp + emptyDir: {} + + extraVolumeMounts: + - name: haproxy-socket + mountPath: /var/run/haproxy + + containerPorts: + http: 4000 + prometheus: 8404 + + service: + type: ClusterIP + + replicaCount: 3 + + resources: + requests: + cpu: "1" + memory: "512Mi" + limits: + cpu: "4" + memory: "2Gi" + + autoscaling: + enabled: false + + # Enable the HAProxy chart's actual PDB key. The previous lower-case + # podDisruptionBudget block was ignored by upstream chart 1.26.1. + PodDisruptionBudget: + enable: true + minAvailable: 2 + + # Pin HAProxy pods to the on-demand node pool so edge/proxy capacity is not + # removed by abrupt spot interruptions. + nodeSelector: + purpose: on-demand + tolerations: + - key: dedicated + operator: Equal + value: on-demand + effect: NoSchedule + + podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/instance: erpc + app.kubernetes.io/name: erpc-processing-haproxy + topologyKey: kubernetes.io/hostname + + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/instance: erpc + app.kubernetes.io/name: erpc-processing-haproxy + +erpc-router-haproxy: + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/haproxytech/haproxy-alpine + + rbac: + create: true + + serviceMonitor: + enabled: true + extraLabels: + release: prometheus + endpoints: + - port: prometheus + path: /metrics + scheme: http + + config: | + global + stats socket /var/run/haproxy/admin.sock mode 660 level admin expose-fd listeners + stats timeout 30s + maxconn 500000 + spread-checks 5 + log stdout format raw local0 info + ulimit-n 1048576 + tune.bufsize 131072 + tune.maxrewrite 8192 + tune.http.maxhdr 128 + tune.recv_enough 16384 + tune.runqueue-depth 512 + + defaults + mode http + log global + option httplog + option dontlognull + option http-server-close + option forwardfor except 127.0.0.0/8 + timeout connect 5s + timeout client 60s + timeout server 60s + timeout http-request 30s + timeout http-keep-alive 30s + timeout queue 120s + timeout check 15s + retries 3 + option redispatch + retry-on conn-failure empty-response response-timeout 503 + + frontend erpc-router-frontend + bind *:4000 + maxconn 500000 + unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid + unique-id-header X-Request-ID + + # If every router eRPC server is down, bypass the metric debounce and + # route straight to fallback. The controller still handles gradual + # degraded-performance activation through erpc-router-backend weights. + use_backend erpc-router-fallback-backend if { nbsrv(erpc-router-main-health-backend) eq 0 } + + default_backend erpc-router-backend + + backend erpc-router-main-health-backend + balance roundrobin + option httpchk GET /healthcheck + http-check expect status 200 + default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 check + server-template erpc-router-main-health 100 erpc-router-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none + + backend erpc-router-backend + # Weighted round-robin caps fallback traffic by configured weight. + balance roundrobin + option httpchk GET /healthcheck + http-check expect status 200 + http-reuse safe + default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check + server-template erpc-router 100 erpc-router-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none + # Weighted fallback is intentionally capped below fallback HPA max so + # partial fallback traffic cannot grow with scaled-out fallback pods. + server-template erpc-fallback 3 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 weight 0 check resolvers k8s init-addr none + + backend erpc-router-fallback-backend + balance leastconn + option httpchk GET /healthcheck + http-check expect status 200 + http-reuse safe + default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check + server-template erpc-fallback-direct 40 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none + + resolvers k8s + nameserver dns kube-dns.kube-system.svc.cluster.local:53 + accepted_payload_size 8192 + hold valid 5s + hold nx 5s + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + + frontend stats + mode http + bind *:8404 + http-request use-service prometheus-exporter if { path /metrics } + stats enable + stats uri /stats + stats refresh 10s + stats show-legends + stats show-node + + sidecarContainers: + - name: weight-controller + image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/haproxy-controller-erpc:v1.2.2 + imagePullPolicy: IfNotPresent + env: + - name: PROMETHEUS_URL + value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090" + - name: GOROUTINE_QUERY + value: 'erpc:go_goroutines:by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}' + - name: LATENCY_QUERY + value: 'histogram_quantile(0.99, erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}) * 1000' + - name: REQUEST_RATE_QUERY + value: 'erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}' + - name: CPU_QUERY + value: '100 * erpc:container_cpu_usage:limit_ratio1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}' + - name: TARGET_GOROUTINES + value: "400" + - name: TARGET_LATENCY_MS + value: "500" + - name: TARGET_REQUEST_RATE + value: "50" + - name: TARGET_CPU + value: "60" + - name: WEIGHT_GOROUTINES + value: "0.30" + - name: WEIGHT_LATENCY + value: "0.35" + - name: WEIGHT_REQUEST_RATE + value: "0.15" + - name: WEIGHT_CPU + value: "0.20" + - name: BACKEND_NAME + value: "erpc-router-backend" + - name: POD_NAMESPACE + value: "morpho-prd" + - name: UPDATE_INTERVAL + value: "5s" + - name: MIN_WEIGHT + value: "10" + - name: MAX_WEIGHT + value: "50" + - name: HAPROXY_SOCKET + value: "/var/run/haproxy/admin.sock" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "256Mi" + livenessProbe: + exec: + command: + - /bin/sh + - -c + - pgrep -f "controller.sh" > /dev/null + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - test -S "$HAPROXY_SOCKET" + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + volumeMounts: + - name: haproxy-socket + mountPath: /var/run/haproxy + - name: controller-tmp + mountPath: /tmp + - <<: *erpcFallbackController + env: + - name: PROMETHEUS_URL + value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090" + - name: BACKEND_NAME + value: "erpc-router-backend" + - name: MAIN_HEALTH_BACKEND_NAME + value: "erpc-router-main-health-backend" + - name: MAIN_HEALTH_SERVER_PREFIX + value: "erpc-router-main-health" + - name: FALLBACK_SERVER_PREFIX + value: "erpc-fallback" + - name: HAPROXY_SOCKET + value: "/var/run/haproxy/admin.sock" + - name: UPDATE_INTERVAL_SECONDS + value: "30" + - name: ENABLE_AFTER_FAILURES + value: "6" + - name: RECOVER_AFTER_SUCCESSES + value: "20" + - name: DEGRADED_P99_MS + value: "25000" + - name: RECOVERED_P99_MS + value: "10000" + - name: DEGRADED_ERROR_RATE_PERCENT + value: "1" + - name: RECOVERED_ERROR_RATE_PERCENT + value: "0.5" + - name: FALLBACK_ACTIVE_WEIGHT + value: "1" + - name: FALLBACK_IDLE_WEIGHT + value: "0" + - name: MIN_TRAFFIC_RPS + value: "10" + - name: REQUEST_RATE_QUERY + value: 'sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"})' + - name: LATENCY_QUERY + value: 'histogram_quantile(0.99, sum by (le) (erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"})) * 1000' + - name: ERROR_RATE_QUERY + value: '100 * (sum(erpc:network_failed_actionable_request_total:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}) or on() vector(0)) / clamp_min(sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}), 1)' + + extraVolumes: + - name: haproxy-socket + emptyDir: {} + - name: controller-tmp + emptyDir: {} + + extraVolumeMounts: + - name: haproxy-socket + mountPath: /var/run/haproxy + + containerPorts: + http: 4000 + prometheus: 8404 + + service: + type: ClusterIP + + replicaCount: 3 + + resources: + requests: + cpu: "1" + memory: "512Mi" + limits: + cpu: "4" + memory: "2Gi" + + autoscaling: + enabled: false + + # Enable the HAProxy chart's actual PDB key. The previous lower-case + # podDisruptionBudget block was ignored by upstream chart 1.26.1. + PodDisruptionBudget: + enable: true + minAvailable: 2 + + # Pin HAProxy pods to the on-demand node pool so edge/proxy capacity is not + # removed by abrupt spot interruptions. + nodeSelector: + purpose: on-demand + tolerations: + - key: dedicated + operator: Equal + value: on-demand + effect: NoSchedule + + podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/instance: erpc + app.kubernetes.io/name: erpc-router-haproxy + topologyKey: kubernetes.io/hostname + + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/instance: erpc + app.kubernetes.io/name: erpc-router-haproxy + +# ============================================================================= +# eRPC Application Configuration +# ============================================================================= +erpc: + fullnameOverride: "erpc" + + serviceAccount: + runtimeName: "erpc-runtime" + vaultCreator: + create: false + name: "erpc" + + # PLA-1360: enable Pod Security Standards "restricted" hardening on the + # main erpc prd instance after dev validation. The erpc-dev alias is + # already opted in via PR 7536. + securityHardening: + enabled: true + + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc + tag: "0.1.2" + + postgresql: + backup: + method: volumeSnapshot + volumeSnapshotClassName: "csi-aws-vsc" + retentionPolicy: "7d" + destinationPath: "s3://prd-morpho-database-backup/erpc-backups" + # PLA-1112: right-sized from gp3-db-burst (16000/1000) — CloudWatch peak was + # only ~339 IOPS / ~8 MB/s, ~50x under burst. gp3-db-standard (3000/125) + # leaves ample headroom. + storageClass: gp3-db-standard + volumeAttributesClassName: gp3-db-standard + # Primary currently uses ~693Gi; 500Gi chart default would block replica re-bootstrap + storage: "1Ti" + # PLA-1112: 8 CPU / 32Gi forced m8g.4xlarge (capped at 8/32 — paying for a + # 64Gi/16-vCPU node). 7d peak is only ~0.6 CPU cores and ~0.48Gi RSS. + # 7400m/28Gi still rounds up to m8g.4xlarge (both sit at ~98-99% of m8g.2xlarge + # allocatable once DaemonSets are counted), so trim to 6800m / 26Gi to land + # cleanly on m8g.2xlarge with headroom — half the node, ~26Gi cache for the + # ~700GB latency-sensitive DB. + resources: + requests: + cpu: "6800m" + memory: "26Gi" + limits: + cpu: "6800m" + memory: "26Gi" + # Enforce 1-pod-per-node across all CNPG instances (cross-cluster). + affinity: + additionalPodAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: cnpg.io/podRole + operator: In + values: [instance] + topologyKey: kubernetes.io/hostname + sharedBuffers: "8GB" + effectiveCacheSize: "24GB" + maintenanceWorkMem: "1GB" + workMem: "8MB" + maxWorkerProcesses: "4" + maxParallelWorkers: "4" + maxParallelWorkersPerGather: "2" + maxParallelMaintenanceWorkers: "2" + autovacuumMaxWorkers: "4" + + vault: + jobImage: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator + tag: "0.1.2" + + headlessService: + enabled: true + + replicaCount: 10 + + autoscaling: + enabled: true + minReplicas: 10 + maxReplicas: 20 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + + extraEnvVars: + - name: GOMEMLIMIT + value: "6GiB" + - name: GOGC + value: "25" + + resources: + requests: + cpu: "750m" + memory: "4Gi" + limits: + cpu: "1500m" + memory: "8Gi" + + startupProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 6 + + livenessProbe: + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + + readinessProbe: + path: /healthcheck + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 2 + + monitoring: + podMonitor: + enabled: true + additionalLabels: + release: prometheus + interval: 5s + scrapeTimeout: 4s + +# ============================================================================= +# eRPC Dev Instance - Single pod, no HPA, no database +# ============================================================================= +erpc-dev: + enabled: true + fullnameOverride: "erpc-dev" + + # PLA-1360: validate Pod Security Standards "restricted" hardening on the + # erpc dev instance (distroless image) before flipping in the other prd + # erpc instances. + securityHardening: + enabled: true + + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc + tag: "0.1.2" + + replicaCount: 1 + + autoscaling: + enabled: false + minReplicas: 1 + + # Disable PostgreSQL (shares the production database via Vault config) + postgresql: + enabled: false + + # Reuse the existing erpc Vault creator identity only for secret materialization; + # runtime pods use erpc-dev-runtime and have no Secret RBAC. + serviceAccount: + runtimeName: "erpc-dev-runtime" + vaultCreator: + create: false + name: "erpc" + + # Dev config path in Vault. Keep this source cache/state-isolated from prod: + # Redis DB/cluster key are dev-only and PostgreSQL cache connector is disabled. + vault: + configPath: "secret/data/erpc/config-dev" + jobImage: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator + tag: "0.1.2" + pullPolicy: IfNotPresent + + # Public ingress for erpc-dev is managed in morpho-infra via CloudFront and + # a Terraform-owned internal NLB. + service: + type: ClusterIP + + extraEnvVars: + - name: GOMEMLIMIT + value: "640MiB" + - name: GOGC + value: "50" + + resources: + requests: + cpu: "250m" + memory: "384Mi" + limits: + cpu: "1200m" + memory: "768Mi" + + startupProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + + livenessProbe: + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + + readinessProbe: + path: /healthcheck + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 2 + + monitoring: + podMonitor: + enabled: true + additionalLabels: + release: prometheus + interval: 5s + scrapeTimeout: 4s + +# ============================================================================= +# eRPC Fallback Instance - Hot standby upstream-image pool for incident traffic +# ============================================================================= +erpc-fallback: + enabled: true + fullnameOverride: "erpc-fallback" + + securityHardening: + enabled: true + + image: + # Keep the upstream eRPC image for incident fallback, but pin the digest so + # pod restarts cannot drift to a new upstream build without review. + repository: ghcr.io/erpc/erpc + tag: "0.1.0@sha256:f8a292d5e867cf878e5fad1c3b0702531bbf86f3f1d7db1af18e34cbcd5b5a6e" + pullPolicy: IfNotPresent + + autoscaling: + enabled: true + # Keep fallback cheap while idle, but let it burst to full failover capacity + # as soon as real traffic or resource pressure appears. + minReplicas: 3 + maxReplicas: 40 + targetCPUUtilizationPercentage: 50 + targetMemoryUtilizationPercentage: 70 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 200 + periodSeconds: 15 + - type: Pods + value: 16 + periodSeconds: 15 + selectPolicy: Max + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 50 + periodSeconds: 60 + - type: Pods + value: 8 + periodSeconds: 60 + selectPolicy: Max + + postgresql: + enabled: false + + serviceAccount: + runtimeName: "erpc-fallback-runtime" + vaultCreator: + create: false + name: "erpc" + + vault: + configPath: "secret/data/erpc/config-fallback" + secretsPath: "secret/data/erpc/secrets" + jobImage: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator + tag: "0.0.83" + pullPolicy: IfNotPresent + validationImage: + enabled: true + repository: ghcr.io/erpc/erpc + tag: "0.1.0@sha256:f8a292d5e867cf878e5fad1c3b0702531bbf86f3f1d7db1af18e34cbcd5b5a6e" + pullPolicy: IfNotPresent + command: + - /erpc-server + + headlessService: + enabled: true + + extraEnvVars: + - name: GOMEMLIMIT + value: "6GiB" + - name: GOGC + value: "25" + + resources: + requests: + cpu: "250m" + memory: "1Gi" + limits: + cpu: "1500m" + memory: "8Gi" + + lifecycle: + enabled: false + + startupProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + + livenessProbe: + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + + readinessProbe: + path: /healthcheck + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 2 + + monitoring: + podMonitor: + enabled: true + additionalLabels: + release: prometheus + interval: 5s + scrapeTimeout: 4s + +# ============================================================================= +# eRPC Processing Instance - Dedicated pool for heavier workloads +# ============================================================================= +erpc-processing: + enabled: true + fullnameOverride: "erpc-processing" + + # PLA-1360: enable Pod Security Standards "restricted" hardening after + # dev validation. + securityHardening: + enabled: true + + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc + tag: "0.1.2" + + replicaCount: 3 + + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 8 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + + # Disable PostgreSQL (shares the production database via Vault config) + postgresql: + enabled: false + + # Reuse the existing erpc Vault creator identity only for secret materialization; + # runtime pods use erpc-processing-runtime and have no Secret RBAC. + serviceAccount: + runtimeName: "erpc-processing-runtime" + vaultCreator: + create: false + name: "erpc" + + # Dedicated processing config path in Vault + vault: + configPath: "secret/data/erpc/config-processing" + jobImage: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator + tag: "0.1.2" + pullPolicy: IfNotPresent + + extraEnvVars: + - name: GOMEMLIMIT + value: "768MiB" + - name: GOGC + value: "25" + + resources: + requests: + cpu: "750m" + memory: "512Mi" + limits: + cpu: "1500m" + memory: "1Gi" + + headlessService: + enabled: true + + startupProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 6 + + livenessProbe: + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + + readinessProbe: + path: /healthcheck + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 2 + + monitoring: + podMonitor: + enabled: true + additionalLabels: + release: prometheus + interval: 5s + scrapeTimeout: 4s + +# ============================================================================= +# eRPC Router Instance - Dedicated pool for router workloads +# ============================================================================= +erpc-router: + enabled: true + fullnameOverride: "erpc-router" + + # PLA-1360: enable Pod Security Standards "restricted" hardening after + # dev validation. + securityHardening: + enabled: true + + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc + tag: "0.1.2" + + replicaCount: 3 + + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 8 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + + # Disable PostgreSQL (shares the production database via Vault config) + postgresql: + enabled: false + + # Reuse the existing erpc Vault creator identity only for secret materialization; + # runtime pods use erpc-router-runtime and have no Secret RBAC. + serviceAccount: + runtimeName: "erpc-router-runtime" + vaultCreator: + create: false + name: "erpc" + + # Dedicated router config path in Vault + vault: + configPath: "secret/data/erpc/config-router" + jobImage: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator + tag: "0.1.2" + pullPolicy: IfNotPresent + + extraEnvVars: + - name: GOMEMLIMIT + value: "768MiB" + - name: GOGC + value: "25" + + resources: + requests: + cpu: "750m" + memory: "512Mi" + limits: + cpu: "1500m" + memory: "1Gi" + + headlessService: + enabled: true + + startupProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 6 + + livenessProbe: + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + + readinessProbe: + path: /healthcheck + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 2 + + monitoring: + podMonitor: + enabled: true + additionalLabels: + release: prometheus + interval: 5s + scrapeTimeout: 4s + +redis-ha: + enabled: true + fullnameOverride: "erpc-redis" + replicas: 3 + + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/library/redis + tag: 8.4.0-alpine + + configmapTest: + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/koalaman/shellcheck + + # Authentication - uses existing secret created from Vault + auth: true + existingSecret: "erpc-redis-auth" + authKey: "password" + + # Vault configuration for Redis password + vault: + secretsPath: "secret/data/erpc/redis" + + # PodDisruptionBudget - Prevent simultaneous eviction during node scaling + podDisruptionBudget: + minAvailable: 2 # Always keep at least 2 of 3 Redis pods running + + # Hard anti-affinity ensures pods spread across nodes + hardAntiAffinity: true + + # Graceful shutdown - ensure Redis has time to sync before termination + terminationGracePeriodSeconds: 60 + + # Increase replication backlog for faster resync after restarts + redis: + # Custom Redis configuration + # Larger replication backlog prevents full resync after short disconnections + # Default is 1MB, we increase to 256MB to handle longer network partitions + config: + repl-backlog-size: "256mb" + repl-backlog-ttl: "3600" + # Reduce replication timeout for faster failover detection + repl-timeout: "60" + # Ensure RDB persistence for faster recovery + save: "900 1 300 10 60 10000" + resources: + requests: + cpu: "100m" + memory: "512Mi" + limits: + cpu: "1" + memory: "1Gi" + # PreStopHook for graceful failover + # Note: The redis-ha chart has a default trigger-failover-if-master.sh script + # We add BGSAVE before the failover to ensure data is persisted + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - | + # Use REDISCLI_AUTH to avoid password in process args + timeout 10 sh -c 'REDISCLI_AUTH="$AUTH" redis-cli BGSAVE' 2>/dev/null || true + sleep 2 + if [ -f /readonly-config/trigger-failover-if-master.sh ]; then + /bin/sh /readonly-config/trigger-failover-if-master.sh + else + echo "Warning: trigger-failover-if-master.sh not found, skipping failover" + fi + + haproxy: + enabled: true + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/library/haproxy + # Enable Prometheus Redis monitoring via annotation-based discovery + service: + annotations: + prometheus.io/redis-scrape: "true" + prometheus.io/redis-port: "6379" + resources: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" + + sentinel: + resources: + requests: + cpu: "10m" + memory: "32Mi" + limits: + cpu: "100m" + memory: "64Mi" + + persistentVolume: + enabled: true + size: 1Gi + storageClass: "gp3" + + securityContext: + runAsUser: 1000 + fsGroup: 1000 + runAsNonRoot: true + + # Explicit on-demand node placement + nodeSelector: + nodegroup: on-demand-nodes + + tolerations: + - key: dedicated + operator: Equal + value: on-demand + effect: NoSchedule + +# ============================================================================= +# Redis Exporter for Prometheus monitoring +# ============================================================================= +redis-exporter: + image: + repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/oliver006/redis_exporter + + # Connect to redis-ha via HAProxy + redisAddress: redis://erpc-redis-haproxy:6379 + + # Use existing secret created by Vault job + auth: + enabled: true + secret: + name: erpc-redis-auth + key: password + + # Enable ServiceMonitor for Prometheus + serviceMonitor: + enabled: true + labels: + release: prometheus-stack + # Add redis_instance label for Grafana dashboard compatibility + metricRelabelings: + - action: replace + sourceLabels: [service] + targetLabel: redis_instance + regex: (.*) + replacement: erpc-redis + + # Resources + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 25m + memory: 32Mi diff --git a/review/repo-map.md b/review/repo-map.md index ab7203c4d..98963b78f 100644 --- a/review/repo-map.md +++ b/review/repo-map.md @@ -15,28 +15,25 @@ Top-level - .semrelrc - .vscode - AGENTS.md -- CLA.md -- CLAUDE.md -- CODE_OF_CONDUCT.md -- CONTRIBUTING.md -- Dockerfile -- Dockerfile.erpc-validator -- Dockerfile.fake -- Dockerfile.validator -- LICENSE -- Makefile -- README.md - architecture - artifacts - assets - auth - biome.json +- CLA.md +- CLAUDE.md - clients - cmd +- CODE_OF_CONDUCT.md - common - consensus +- CONTRIBUTING.md - data - docker-compose.yml +- Dockerfile +- Dockerfile.erpc-validator +- Dockerfile.fake +- Dockerfile.validator - docs - erpc - erpc.dist.ts @@ -46,13 +43,17 @@ Top-level - go.mod - go.sum - health +- helm - internal - kube +- LICENSE +- Makefile - monitoring - package.json - pnpm-lock.yaml - pnpm-workspace.yaml - prometheus.yaml +- README.md - review - scripts - scylla @@ -65,6 +66,7 @@ Top-level - typescript - upstream - util +- validate-helm-charts.sh Go entrypoints - cmd/erpc diff --git a/validate-helm-charts.sh b/validate-helm-charts.sh new file mode 100755 index 000000000..242f56121 --- /dev/null +++ b/validate-helm-charts.sh @@ -0,0 +1,230 @@ +#!/bin/bash +set -eo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Parse command line arguments (PARALLEL_JOBS set after chart discovery) +PARALLEL_JOBS="" +while getopts "j:sh" opt; do + case $opt in + j) PARALLEL_JOBS="$OPTARG" ;; + s) SEQUENTIAL=true ;; + h) + echo "Usage: $0 [-j N] [-s] [-h]" + echo " -j N Run N parallel jobs (default: number of charts)" + echo " -s Run sequentially (disable parallelism)" + echo " -h Show this help" + exit 0 + ;; + \?) echo "Invalid option: -$OPTARG" >&2; exit 1 ;; + esac +done + +# Check if kubeconform is installed +if ! command -v kubeconform &> /dev/null; then + echo -e "${RED}kubeconform is not installed. Please install it first.${NC}" + echo "You can install it via: https://github.com/yannh/kubeconform#installation" + exit 1 +fi + +# Check if helm is installed +if ! command -v helm &> /dev/null; then + echo -e "${RED}helm is not installed. Please install it first.${NC}" + echo "You can install it via: https://helm.sh/docs/intro/install/" + exit 1 +fi + +# Store the initial working directory +INITIAL_DIR=$(pwd) +export INITIAL_DIR + +# Set base directories to scan (erpc: base chart + prd env wrapper only) +DIRS_TO_SCAN=("helm/charts/erpc" "helm/environments/prd/erpc") + +# List of custom resources to skip +CUSTOM_RESOURCES="PodMonitor,ServiceMonitor,AlertManager,Prometheus,PrometheusRule,Cluster,ScheduledBackup,ImageCatalog,IngressRoute,Certificate,CronJob,EventSource,EventBus,Sensor" +export CUSTOM_RESOURCES + +# List of charts to explicitly skip validation +SKIP_VALIDATION_CHARTS=() + +# Function to check if a chart should skip validation +should_skip_validation() { + local chart_name=$1 + + # Check explicit list + for skip_chart in "${SKIP_VALIDATION_CHARTS[@]}"; do + if [[ "$chart_name" == "$skip_chart" ]]; then + return 0 # true, should skip + fi + done + + # Check database pattern + if [[ "$chart_name" == *"-db" || "$chart_name" == *"database"* || "$chart_name" == *"postgres"* ]]; then + return 0 # true, should skip + fi + + return 1 # false, should not skip +} + +# Export for parallel subshells +export -f should_skip_validation +export SKIP_VALIDATION_CHARTS + +# Function to process a single chart (for parallel execution) +process_chart() { + local chart_path=$1 + local chart_name=$(basename "$chart_path") + local result_file=$(mktemp) + local has_error=0 + + # Store the absolute path to the chart + local chart_absolute_path=$(realpath "$chart_path") + + echo "Processing: ${chart_path}" >&2 + + # Run helm lint + if ! helm lint "$chart_absolute_path" > "$result_file.lint" 2>&1; then + echo -e "${RED}✗ Helm lint failed for ${chart_name}${NC}" >&2 + cat "$result_file.lint" >&2 + has_error=1 + else + echo -e "${GREEN}✓ Helm lint passed for ${chart_name}${NC}" >&2 + fi + + # Check if this chart should skip validation + if should_skip_validation "$chart_name"; then + echo -e "${YELLOW}⊘ Skipping kubeconform for ${chart_name} (database chart)${NC}" >&2 + rm -f "$result_file" "$result_file.lint" "$result_file.template" + exit $has_error + fi + + # Build dependencies if Chart.yaml exists and has dependencies + if [ -f "${chart_absolute_path}/Chart.yaml" ]; then + if grep -q "dependencies:" "${chart_absolute_path}/Chart.yaml"; then + cd "$chart_absolute_path" + if ! helm dependency update > /dev/null 2>&1; then + echo -e "${RED}✗ Dependency update failed for ${chart_name}${NC}" >&2 + cd "$INITIAL_DIR" + rm -f "$result_file" "$result_file.lint" "$result_file.template" + exit 1 + fi + fi + + # Run template validation + cd "$chart_absolute_path" + if ! helm template . 2>/dev/null | kubeconform --ignore-missing-schemas --summary --skip "$CUSTOM_RESOURCES" > "$result_file.template" 2>&1; then + echo -e "${RED}✗ Kubeconform failed for ${chart_name}${NC}" >&2 + cat "$result_file.template" >&2 + has_error=1 + else + echo -e "${GREEN}✓ Kubeconform passed for ${chart_name}${NC}" >&2 + fi + cd "$INITIAL_DIR" + else + echo -e "${RED}No Chart.yaml found in ${chart_path}${NC}" >&2 + has_error=1 + fi + + rm -f "$result_file" "$result_file.lint" "$result_file.template" + exit $has_error +} + +# Export for parallel +export -f process_chart + +# Main script execution +echo -e "${YELLOW}Starting Helm chart validation...${NC}" +echo -e "${YELLOW}Skipping validation for these custom resources: ${CUSTOM_RESOURCES}${NC}" + +# Add common repositories that might be needed +echo -e "${YELLOW}Adding common Helm repositories...${NC}" +helm repo add bitnami https://charts.bitnami.com/bitnami 2>/dev/null || true +helm repo add cnpg https://cloudnative-pg.github.io/charts 2>/dev/null || true +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 2>/dev/null || true +helm repo add haproxytech https://haproxytech.github.io/helm-charts 2>/dev/null || true +helm repo add dandydeveloper https://dandydeveloper.github.io/charts 2>/dev/null || true + +# Update Helm repositories +echo -e "${YELLOW}Updating Helm repositories...${NC}" +helm repo update + +# Collect all charts to validate +CHARTS=() +for dir in "${DIRS_TO_SCAN[@]}"; do + if [ ! -d "$dir" ]; then + echo -e "${YELLOW}Directory $dir does not exist, skipping...${NC}" + continue + fi + + while IFS= read -r chart_yaml; do + chart_dir=$(dirname "$chart_yaml") + CHARTS+=("$chart_dir") + done < <(find "$dir" -name Chart.yaml -type f) +done + +TOTAL_CHARTS=${#CHARTS[@]} + +# Set PARALLEL_JOBS to number of charts if not specified +if [ -z "$PARALLEL_JOBS" ]; then + PARALLEL_JOBS=$TOTAL_CHARTS +fi + +echo -e "\n${YELLOW}Found ${TOTAL_CHARTS} charts to validate${NC}\n" + +# Run validation +FAILED_CHARTS=() + +if [ "$SEQUENTIAL" = true ] || ! command -v parallel &> /dev/null; then + # Sequential execution (fallback or explicit) + if [ "$SEQUENTIAL" != true ]; then + echo -e "${YELLOW}GNU parallel not found, falling back to sequential execution${NC}" + echo -e "${YELLOW}Install with: brew install parallel (macOS) or apt install parallel (Linux)${NC}\n" + fi + + for chart in "${CHARTS[@]}"; do + if ! process_chart "$chart"; then + FAILED_CHARTS+=("$chart") + fi + done +else + # Parallel execution + echo -e "${BLUE}Running ${PARALLEL_JOBS} parallel validation jobs...${NC}\n" + + # Create a temporary file to track failures + FAIL_FILE=$(mktemp) + + # Run in parallel, collecting failures + printf '%s\n' "${CHARTS[@]}" | \ + parallel -j "$PARALLEL_JOBS" --halt never,fail=1 --line-buffer \ + "process_chart {} || echo {} >> $FAIL_FILE" + + # Collect failed charts + if [ -f "$FAIL_FILE" ]; then + while IFS= read -r failed; do + FAILED_CHARTS+=("$failed") + done < "$FAIL_FILE" + rm -f "$FAIL_FILE" + fi +fi + +# Summary report +echo -e "\n${YELLOW}==== Validation Summary ====${NC}" +echo -e "Total charts processed: ${TOTAL_CHARTS}" +echo -e "Failed charts: ${#FAILED_CHARTS[@]}" + +if [ ${#FAILED_CHARTS[@]} -eq 0 ]; then + echo -e "${GREEN}All validations passed successfully!${NC}" + exit 0 +else + echo -e "${RED}The following charts failed validation:${NC}" + for chart in "${FAILED_CHARTS[@]}"; do + echo -e " - ${chart}" + done + exit 1 +fi From 373fdfecfcecfcb2ef47b56ac9e83f0482f50c7f Mon Sep 17 00:00:00 2001 From: rguichard Date: Thu, 25 Jun 2026 12:09:58 +0200 Subject: [PATCH 02/12] fix(helm): address Aikido review findings on validation scripts Apply the fixes from the per-finding review branches directly: validate-helm-charts.sh: - process_chart now returns instead of exit, so the sequential loop validates every chart and reaches the summary (the parallel path is unaffected). - Use a single `mktemp -d` dir (lint/template files) instead of an unused base temp file; clean up with one `rm -rf`. - Pass the chart path to `helm dependency update` / `helm template` instead of cd-ing into the dir; drop the now-unused INITIAL_DIR. - Split the `helm template | kubeconform` pipeline for readability (keeping the `if !` guard, which is pipefail-safe). test-local.sh: - Redact LABELS_SERVICE_API_KEY when echoing the auth response. - Correct the eth_getLogs decimal block numbers (0x1254048f = 307,496,079, 0x12558b2f = 307,596,079, range = 100,000). The Aikido suggestion to move should_skip_validation before `helm lint` was intentionally not applied: lint is meant to run for all charts; only kubeconform is skipped for database charts. Co-Authored-By: Claude Opus 4.8 --- helm/charts/erpc/test-local.sh | 8 ++++--- validate-helm-charts.sh | 40 ++++++++++++++++------------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/charts/erpc/test-local.sh b/helm/charts/erpc/test-local.sh index 164658283..e9d3b1ecf 100755 --- a/helm/charts/erpc/test-local.sh +++ b/helm/charts/erpc/test-local.sh @@ -192,7 +192,9 @@ if [ -n "${LABELS_SERVICE_API_KEY:-}" ] && [ "$LABELS_SERVICE_API_KEY" != "your- echo "✅ Authentication with proper secret parameter works" else echo "⚠️ Authentication secret parameter didn't work as expected" - echo " Response: $auth_response_with_secret" + # Redact the API key in case the response (or a reflected error) + # echoes back the request URL containing the secret. + echo " Response: ${auth_response_with_secret//$LABELS_SERVICE_API_KEY/***REDACTED***}" fi else echo "⚠️ Authentication might not be properly configured (no auth rejection detected)" @@ -222,8 +224,8 @@ if echo "$response" | grep -q '"result"'; then log_count=$(echo "$response" | jq -r '.result | length' 2>/dev/null || echo "unknown") echo "✅ Arbitrum eth_getLogs test successful" echo " Logs found: $log_count" - echo " Block range: 0x1254048f (307,074,191) to 0x12558b2f (307,166,767)" - echo " Range size: ~92,576 blocks" + echo " Block range: 0x1254048f (307,496,079) to 0x12558b2f (307,596,079)" + echo " Range size: 100,000 blocks" elif echo "$response" | grep -q '"error"'; then error_message=$(echo "$response" | jq -r '.error.message // .error' 2>/dev/null || echo "unknown") echo "⚠️ eth_getLogs returned error (expected for large ranges):" diff --git a/validate-helm-charts.sh b/validate-helm-charts.sh index 242f56121..a533802d1 100755 --- a/validate-helm-charts.sh +++ b/validate-helm-charts.sh @@ -39,10 +39,6 @@ if ! command -v helm &> /dev/null; then exit 1 fi -# Store the initial working directory -INITIAL_DIR=$(pwd) -export INITIAL_DIR - # Set base directories to scan (erpc: base chart + prd env wrapper only) DIRS_TO_SCAN=("helm/charts/erpc" "helm/environments/prd/erpc") @@ -80,7 +76,9 @@ export SKIP_VALIDATION_CHARTS process_chart() { local chart_path=$1 local chart_name=$(basename "$chart_path") - local result_file=$(mktemp) + local result_dir=$(mktemp -d) + local lint_file="$result_dir/lint" + local template_file="$result_dir/template" local has_error=0 # Store the absolute path to the chart @@ -89,9 +87,9 @@ process_chart() { echo "Processing: ${chart_path}" >&2 # Run helm lint - if ! helm lint "$chart_absolute_path" > "$result_file.lint" 2>&1; then + if ! helm lint "$chart_absolute_path" > "$lint_file" 2>&1; then echo -e "${RED}✗ Helm lint failed for ${chart_name}${NC}" >&2 - cat "$result_file.lint" >&2 + cat "$lint_file" >&2 has_error=1 else echo -e "${GREEN}✓ Helm lint passed for ${chart_name}${NC}" >&2 @@ -100,39 +98,39 @@ process_chart() { # Check if this chart should skip validation if should_skip_validation "$chart_name"; then echo -e "${YELLOW}⊘ Skipping kubeconform for ${chart_name} (database chart)${NC}" >&2 - rm -f "$result_file" "$result_file.lint" "$result_file.template" - exit $has_error + rm -rf "$result_dir" + return $has_error fi # Build dependencies if Chart.yaml exists and has dependencies if [ -f "${chart_absolute_path}/Chart.yaml" ]; then if grep -q "dependencies:" "${chart_absolute_path}/Chart.yaml"; then - cd "$chart_absolute_path" - if ! helm dependency update > /dev/null 2>&1; then + if ! helm dependency update "$chart_absolute_path" > /dev/null 2>&1; then echo -e "${RED}✗ Dependency update failed for ${chart_name}${NC}" >&2 - cd "$INITIAL_DIR" - rm -f "$result_file" "$result_file.lint" "$result_file.template" - exit 1 + rm -rf "$result_dir" + return 1 fi fi - # Run template validation - cd "$chart_absolute_path" - if ! helm template . 2>/dev/null | kubeconform --ignore-missing-schemas --summary --skip "$CUSTOM_RESOURCES" > "$result_file.template" 2>&1; then + # Run template validation: render the chart and validate the + # resulting manifests with kubeconform. The pipeline's exit status + # reflects kubeconform (the last command), which is what we check. + if ! helm template "$chart_absolute_path" 2>/dev/null \ + | kubeconform --ignore-missing-schemas --summary --skip "$CUSTOM_RESOURCES" \ + > "$template_file" 2>&1; then echo -e "${RED}✗ Kubeconform failed for ${chart_name}${NC}" >&2 - cat "$result_file.template" >&2 + cat "$template_file" >&2 has_error=1 else echo -e "${GREEN}✓ Kubeconform passed for ${chart_name}${NC}" >&2 fi - cd "$INITIAL_DIR" else echo -e "${RED}No Chart.yaml found in ${chart_path}${NC}" >&2 has_error=1 fi - rm -f "$result_file" "$result_file.lint" "$result_file.template" - exit $has_error + rm -rf "$result_dir" + return $has_error } # Export for parallel From a8d49f20910feb32f78a29a18db6ae67ee9b7fc5 Mon Sep 17 00:00:00 2001 From: rguichard Date: Thu, 25 Jun 2026 12:17:37 +0200 Subject: [PATCH 03/12] chore(helm): address @0x666c6f review on CODEOWNERS and image versions - CODEOWNERS: add @0x666c6f as a mandatory codeowner alongside the teams, on every pattern (incl. /helm/) so he is a required reviewer repo-wide, not only for files matching the default `*` rule. - values.yaml: bump erpc and erpc-validator image tags 0.0.77 -> 0.1.3 to match the latest prd values in morpho-org/morpho-infra-helm. Co-Authored-By: Claude Opus 4.8 --- .github/CODEOWNERS | 8 ++++---- helm/charts/erpc/values.yaml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d5985d7fd..1b3c6e201 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,7 @@ # Default ownership -* @morpho-org/platform-engineers +* @morpho-org/platform-engineers @0x666c6f # Helm charts migrated from morpho-infra-helm (2026-06-17) -/helm/ @morpho-org/platform-engineers @morpho-org/api-engineers -/validate-helm-charts.sh @morpho-org/platform-engineers -/setup-hooks.sh @morpho-org/platform-engineers +/helm/ @morpho-org/platform-engineers @morpho-org/api-engineers @0x666c6f +/validate-helm-charts.sh @morpho-org/platform-engineers @0x666c6f +/setup-hooks.sh @morpho-org/platform-engineers @0x666c6f diff --git a/helm/charts/erpc/values.yaml b/helm/charts/erpc/values.yaml index d85b55494..3f290978f 100644 --- a/helm/charts/erpc/values.yaml +++ b/helm/charts/erpc/values.yaml @@ -12,7 +12,7 @@ vault: secretsPath: "secret/data/erpc/secrets" jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/erpc-validator - tag: "0.0.77" + tag: "0.1.3" pullPolicy: IfNotPresent jobResources: requests: @@ -38,7 +38,7 @@ vault: image: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/erpc - tag: "0.0.77" + tag: "0.1.3" pullPolicy: IfNotPresent imagePullSecrets: [] From 7dd92d7ecdec2b61a8ee8525f3bf22d1cc06eb4c Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 29 Jun 2026 13:59:12 +0200 Subject: [PATCH 04/12] fix(harness): make repo map sort deterministic --- scripts/review-repo-map.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/review-repo-map.sh b/scripts/review-repo-map.sh index 13a7b21b5..ad7cfc1a2 100755 --- a/scripts/review-repo-map.sh +++ b/scripts/review-repo-map.sh @@ -19,7 +19,7 @@ echo echo "Top-level" if [[ $in_git -eq 1 ]]; then - git ls-files | awk -F/ '{print $1}' | sort -u | sed 's/^/- /' + git ls-files | awk -F/ '{print $1}' | LC_ALL=C sort -f -u | sed 's/^/- /' else ls -1 | sed 's/^/- /' fi From 75afe9075327999954dc5d5f245b30d848c87235 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 29 Jun 2026 14:04:09 +0200 Subject: [PATCH 05/12] ci(helm): stop persisting checkout credentials --- .github/workflows/helm-charts-validation.yaml | 4 ++++ .github/workflows/wiz-iac-scan.yml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/helm-charts-validation.yaml b/.github/workflows/helm-charts-validation.yaml index 17b13db14..49a93b59e 100644 --- a/.github/workflows/helm-charts-validation.yaml +++ b/.github/workflows/helm-charts-validation.yaml @@ -23,6 +23,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - name: Find all charts id: find @@ -48,6 +50,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - name: Set up Helm uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4 diff --git a/.github/workflows/wiz-iac-scan.yml b/.github/workflows/wiz-iac-scan.yml index a16ddc108..6e2df7148 100644 --- a/.github/workflows/wiz-iac-scan.yml +++ b/.github/workflows/wiz-iac-scan.yml @@ -21,6 +21,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - name: Download Wiz CLI run: | From ef0e193bea9a18a4c25328d639331264dc6294c2 Mon Sep 17 00:00:00 2001 From: rguichard Date: Mon, 6 Jul 2026 16:36:59 +0200 Subject: [PATCH 06/12] chore(helm): resync prd erpc values from morpho-infra-helm HEAD The import predates the latest bumps in morpho-infra-helm (images 0.1.3, main replicaCount 12); cutting ArgoCD over on the stale copy would roll production back. --- helm/environments/prd/erpc/values.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/helm/environments/prd/erpc/values.yaml b/helm/environments/prd/erpc/values.yaml index 3cc75d4ef..f2d737ff3 100644 --- a/helm/environments/prd/erpc/values.yaml +++ b/helm/environments/prd/erpc/values.yaml @@ -1330,7 +1330,7 @@ erpc: image: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc - tag: "0.1.2" + tag: "0.1.3" postgresql: backup: @@ -1381,16 +1381,16 @@ erpc: vault: jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator - tag: "0.1.2" + tag: "0.1.3" headlessService: enabled: true - replicaCount: 10 + replicaCount: 12 autoscaling: enabled: true - minReplicas: 10 + minReplicas: 12 maxReplicas: 20 targetCPUUtilizationPercentage: 80 targetMemoryUtilizationPercentage: 80 @@ -1449,7 +1449,7 @@ erpc-dev: image: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc - tag: "0.1.2" + tag: "0.1.3" replicaCount: 1 @@ -1475,7 +1475,7 @@ erpc-dev: configPath: "secret/data/erpc/config-dev" jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator - tag: "0.1.2" + tag: "0.1.3" pullPolicy: IfNotPresent # Public ingress for erpc-dev is managed in morpho-infra via CloudFront and @@ -1585,7 +1585,7 @@ erpc-fallback: secretsPath: "secret/data/erpc/secrets" jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator - tag: "0.0.83" + tag: "0.1.3" pullPolicy: IfNotPresent validationImage: enabled: true @@ -1656,7 +1656,7 @@ erpc-processing: image: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc - tag: "0.1.2" + tag: "0.1.3" replicaCount: 3 @@ -1684,7 +1684,7 @@ erpc-processing: configPath: "secret/data/erpc/config-processing" jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator - tag: "0.1.2" + tag: "0.1.3" pullPolicy: IfNotPresent extraEnvVars: @@ -1743,7 +1743,7 @@ erpc-router: image: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc - tag: "0.1.2" + tag: "0.1.3" replicaCount: 3 @@ -1771,7 +1771,7 @@ erpc-router: configPath: "secret/data/erpc/config-router" jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator - tag: "0.1.2" + tag: "0.1.3" pullPolicy: IfNotPresent extraEnvVars: From 9d83bdf7fe3e20008084e1e857d76494401d91ca Mon Sep 17 00:00:00 2001 From: rguichard Date: Mon, 6 Jul 2026 16:37:36 +0200 Subject: [PATCH 07/12] chore(helm): anchor erpc image tags for automated bumps Anchor the nine ECR image tags (erpcImageTag for the erpc runtime, erpcValidatorImageTag for the validator job image) so the update-image-tag workflow can bump them all in one place. The upstream ghcr.io digest pins are intentionally left out of the anchors. --- helm/environments/prd/erpc/values.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/helm/environments/prd/erpc/values.yaml b/helm/environments/prd/erpc/values.yaml index f2d737ff3..3c3b0d59c 100644 --- a/helm/environments/prd/erpc/values.yaml +++ b/helm/environments/prd/erpc/values.yaml @@ -1330,7 +1330,7 @@ erpc: image: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc - tag: "0.1.3" + tag: &erpcImageTag "0.1.3" postgresql: backup: @@ -1381,7 +1381,7 @@ erpc: vault: jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator - tag: "0.1.3" + tag: &erpcValidatorImageTag "0.1.3" headlessService: enabled: true @@ -1449,7 +1449,7 @@ erpc-dev: image: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc - tag: "0.1.3" + tag: *erpcImageTag replicaCount: 1 @@ -1475,7 +1475,7 @@ erpc-dev: configPath: "secret/data/erpc/config-dev" jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator - tag: "0.1.3" + tag: *erpcValidatorImageTag pullPolicy: IfNotPresent # Public ingress for erpc-dev is managed in morpho-infra via CloudFront and @@ -1585,7 +1585,7 @@ erpc-fallback: secretsPath: "secret/data/erpc/secrets" jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator - tag: "0.1.3" + tag: *erpcValidatorImageTag pullPolicy: IfNotPresent validationImage: enabled: true @@ -1656,7 +1656,7 @@ erpc-processing: image: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc - tag: "0.1.3" + tag: *erpcImageTag replicaCount: 3 @@ -1684,7 +1684,7 @@ erpc-processing: configPath: "secret/data/erpc/config-processing" jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator - tag: "0.1.3" + tag: *erpcValidatorImageTag pullPolicy: IfNotPresent extraEnvVars: @@ -1743,7 +1743,7 @@ erpc-router: image: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc - tag: "0.1.3" + tag: *erpcImageTag replicaCount: 3 @@ -1771,7 +1771,7 @@ erpc-router: configPath: "secret/data/erpc/config-router" jobImage: repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator - tag: "0.1.3" + tag: *erpcValidatorImageTag pullPolicy: IfNotPresent extraEnvVars: From a9e6149c57af8a97b977031aea536357bb96aea8 Mon Sep 17 00:00:00 2001 From: rguichard Date: Mon, 6 Jul 2026 16:39:25 +0200 Subject: [PATCH 08/12] ci(helm): add update-image-tag workflow and values.yaml guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replicate the morpho-api in-repo image bump flow, adapted to erpc: - update-image-tag.yaml: workflow_dispatch with a semver version_tag (validated), prd/morpho-main only (erpc has no dev environment or branch), bumps the erpcImageTag and erpcValidatorImageTag anchors, creates a GitHub-signed commit via createCommitOnBranch with the morpho-infra-deployer app token, opens a PR and — when vars.ERPC_PRD_AUTOMERGE is true — waits for required checks with the workflow token then merges with admin bypass. Auto-merge defaults to off: production PRs require manual approval. - only-values-yaml-guard.yml: same guard as morpho-api targeting morpho-main, scoping the bot's review bypass to values.yaml-only changes. --- .github/workflows/only-values-yaml-guard.yml | 63 ++++ .github/workflows/update-image-tag.yaml | 291 +++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 .github/workflows/only-values-yaml-guard.yml create mode 100644 .github/workflows/update-image-tag.yaml diff --git a/.github/workflows/only-values-yaml-guard.yml b/.github/workflows/only-values-yaml-guard.yml new file mode 100644 index 000000000..2251b5a45 --- /dev/null +++ b/.github/workflows/only-values-yaml-guard.yml @@ -0,0 +1,63 @@ +# Required status check that scopes the morpho-infra-deployer bot's review +# bypass to values.yaml-only changes: it is skipped unless the deployer bot +# authored the PR or triggered the run, and fails only when such a PR touches +# anything other than values.yaml. +name: only-values-yaml-guard + +on: + pull_request: + branches: [morpho-main] + merge_group: + +permissions: + contents: read + pull-requests: read + +jobs: + guard: + name: values-yaml-guard + runs-on: ubuntu-latest + # Skipped for non-bot PRs: a skipped job SATISFIES required status checks, + # so human PRs merge normally without spinning up a runner. Covers the bot + # both as PR author and as pusher on someone else's PR, matching on the + # immutable user id (not the login) so an app rename cannot silently turn + # the guard off while the bot keeps its review-bypass. + if: github.event.pull_request.user.id == 187494245 || github.actor_id == 187494245 # morpho-infra-deployer[bot] + steps: + - name: Enforce values.yaml-only changes for the deployer bot + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + MERGE_GROUP_HEAD_REF: ${{ github.event.merge_group.head_ref }} + REPO: ${{ github.repository }} + run: | + # merge_group events carry no pull_request.number; extract it from + # the head ref (refs/heads/gh-readonly-queue//pr--). + if [[ -z "$PR_NUMBER" && -n "$MERGE_GROUP_HEAD_REF" ]]; then + PR_NUMBER=$(echo "$MERGE_GROUP_HEAD_REF" | grep -oE '/pr-[0-9]+' | grep -oE '[0-9]+') + fi + + # Include previous_filename so a rename cannot smuggle a non-values + # path out of sight (the files API only reports the new path). + if ! mapfile -t CHANGED < <(gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files" --jq '.[] | .filename, (.previous_filename // empty)'); then + echo "❌ Failed to fetch changed files from the GitHub API." + exit 1 + fi + echo "Files changed by the deployer bot:" + printf ' %s\n' "${CHANGED[@]}" + + OFFENDING=() + for file in "${CHANGED[@]}"; do + if [[ "$file" != "values.yaml" && "$file" != *"/values.yaml" ]]; then + OFFENDING+=("$file") + fi + done + + if (( ${#OFFENDING[@]} > 0 )); then + echo "❌ Deployer bot PR modifies files other than values.yaml:" + printf ' %s\n' "${OFFENDING[@]}" + echo "The bot may only bypass review for values.yaml-only changes." + exit 1 + fi + + echo "✅ Deployer bot PR only modifies values.yaml files." diff --git a/.github/workflows/update-image-tag.yaml b/.github/workflows/update-image-tag.yaml new file mode 100644 index 000000000..9b4a0fdc2 --- /dev/null +++ b/.github/workflows/update-image-tag.yaml @@ -0,0 +1,291 @@ +name: Update chart image tag + +permissions: {} + +on: + workflow_dispatch: + inputs: + version_tag: + description: 'Release version to deploy (e.g., 0.1.3)' + required: true + type: string + release_url: + description: 'Release URL' + required: false + type: string + +jobs: + update-image-tag: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + checks: read + statuses: read + actions: read + env: + INPUT_VERSION_TAG: ${{ inputs.version_tag }} + INPUT_RELEASE_URL: ${{ inputs.release_url }} + ANCHOR_NAMES: 'erpcImageTag erpcValidatorImageTag' + steps: + - name: Validate version + run: | + if [[ ! "$INPUT_VERSION_TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "❌ Invalid version '$INPUT_VERSION_TAG'. Expected semver like 0.1.3" + exit 1 + fi + + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: morpho-main + fetch-depth: 0 + persist-credentials: false + + - name: Determine auto-merge + id: automerge + run: | + # erpc only deploys to prd; auto-merge is opt-in via vars.ERPC_PRD_AUTOMERGE. + if [[ "${{ vars.ERPC_PRD_AUTOMERGE }}" == "true" ]]; then + echo "enabled=true" >> $GITHUB_OUTPUT + echo "✅ Auto-merge enabled" + else + echo "enabled=false" >> $GITHUB_OUTPUT + echo "⏸️ Auto-merge disabled — PR will require manual approval" + fi + + - name: Generate GitHub App Token + id: generate-token + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1 + with: + app-id: ${{ secrets.INFRA_DEPLOYER_APP_ID }} + private-key: ${{ secrets.INFRA_DEPLOYER_SECRET_KEY }} + + - name: Setup Git remote + run: | + git remote set-url origin https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/${{ github.repository }}.git + + - name: Create feature branch + id: create-branch + run: | + PR_BRANCH="update-image-tag/erpc/prd" + + if git ls-remote --heads origin "$PR_BRANCH" | grep -q "$PR_BRANCH"; then + echo "Branch $PR_BRANCH exists remotely, deleting it" + git push origin --delete "$PR_BRANCH" || true + fi + + git checkout -b "$PR_BRANCH" + git push -f origin "$PR_BRANCH" + echo "pr_branch=$PR_BRANCH" >> $GITHUB_OUTPUT + + - name: Update image tags across all values.yaml + id: update + run: | + NEW_VERSION="$INPUT_VERSION_TAG" + + UPDATED=0 + FILES_UPDATED="" + + while IFS= read -r FILE; do + for ANCHOR in $ANCHOR_NAMES; do + if grep -qE "tag: &${ANCHOR} " "$FILE"; then + # Update the anchor value: 'tag: & ""' — the + # aliased (*) occurrences follow automatically. + sed -i.bak -E "s|(tag: &${ANCHOR} )\"[^\"]+\"|\1\"${NEW_VERSION}\"|" "$FILE" + rm -f "${FILE}.bak" + fi + done + + if ! git diff --quiet "$FILE"; then + echo "✓ Updated: $FILE" + git --no-pager diff "$FILE" | head -10 + UPDATED=$((UPDATED + 1)) + FILES_UPDATED="${FILES_UPDATED}- \`${FILE}\`\n" + fi + done < <(find helm/environments/prd -name 'values.yaml' -type f | sort -u) + + echo "Total files updated: $UPDATED" + echo "count=$UPDATED" >> $GITHUB_OUTPUT + { + echo "files_list<> $GITHUB_OUTPUT + + - name: Create signed commit via API + id: commit + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PR_BRANCH: ${{ steps.create-branch.outputs.pr_branch }} + with: + github-token: ${{ steps.generate-token.outputs.token }} + script: | + const { execSync } = require("node:child_process"); + const fs = require("node:fs"); + + const changed = execSync("git diff --name-only -- helm/environments", { encoding: "utf8" }) + .split("\n") + .filter(Boolean); + + if (changed.length === 0) { + core.setOutput("has_changes", "false"); + core.info("No changes to commit — image tags already up to date"); + return; + } + + const version = process.env.INPUT_VERSION_TAG; + const url = process.env.INPUT_RELEASE_URL || "n/a"; + + const additions = changed.map((path) => ({ + path, + contents: fs.readFileSync(path).toString("base64"), + })); + + const expectedHeadOid = execSync("git rev-parse HEAD", { encoding: "utf8" }).trim(); + + const { createCommitOnBranch } = await github.graphql( + `mutation ($input: CreateCommitOnBranchInput!) { + createCommitOnBranch(input: $input) { + commit { oid } + } + }`, + { + input: { + branch: { + repositoryNameWithOwner: `${context.repo.owner}/${context.repo.repo}`, + branchName: process.env.PR_BRANCH, + }, + message: { + headline: `[erpc] 🤖 Update image tags to ${version} in production`, + body: `Updates the erpc chart deployments to use the latest image.\n\nVersion: ${version}\nRelease URL: ${url}`, + }, + fileChanges: { additions }, + expectedHeadOid, + }, + }, + ); + + core.info(`Created signed commit ${createCommitOnBranch.commit.oid}`); + core.setOutput("has_changes", "true"); + core.setOutput("commit_oid", createCommitOnBranch.commit.oid); + + - name: Create or update PR + id: create-or-update-pr + if: steps.commit.outputs.has_changes == 'true' + env: + FILES_LIST: ${{ steps.update.outputs.files_list }} + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + run: | + PR_BRANCH="${{ steps.create-branch.outputs.pr_branch }}" + + if [[ "${{ steps.automerge.outputs.enabled }}" == "true" ]]; then + AUTO_MERGE_STATUS="✅ **Auto-merged** - This PR is automatically merged once its required checks pass." + else + AUTO_MERGE_STATUS="⏸️ **Manual approval required** - This PR requires manual review and approval before merging." + fi + + PR_TITLE="chore(helm): update erpc image tags in production" + PR_BODY="## 🤖 Automated Image Update + + This PR updates all erpc chart deployments in **production** to the latest image. + + ### Files Updated (${{ steps.update.outputs.count }}) + ${FILES_LIST} + + ### Source Information + - Version: ${INPUT_VERSION_TAG} + - Release URL: ${INPUT_RELEASE_URL} + - Anchors: \`${ANCHOR_NAMES}\` + + ### Merge Status + ${AUTO_MERGE_STATUS}" + + # The head branch was deleted and recreated earlier in this run, which + # auto-closes any previous PR. gh pr list relies on the eventually + # consistent search index and can still return that just-closed PR, + # so confirm the state with a strongly consistent direct lookup + # before editing instead of creating. + PR_NUMBER=$(gh pr list --head "$PR_BRANCH" --json number --jq '.[0].number // empty') + if [[ -n "$PR_NUMBER" ]] && [[ "$(gh pr view "$PR_NUMBER" --json state --jq '.state')" == "OPEN" ]]; then + echo "PR #$PR_NUMBER already exists for branch $PR_BRANCH, updating it" + gh pr edit "$PR_NUMBER" --base "morpho-main" --title "$PR_TITLE" --body "$PR_BODY" + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + else + PR_CREATE_ARGS=( + --title "$PR_TITLE" + --body "$PR_BODY" + --base "morpho-main" + --head "$PR_BRANCH" + ) + + if [[ "${{ steps.automerge.outputs.enabled }}" != "true" ]]; then + PR_CREATE_ARGS+=(--draft) + fi + + PR_URL=$(gh pr create "${PR_CREATE_ARGS[@]}") + + PR_NUMBER=$(echo "$PR_URL" | grep -oE '[0-9]+$') + echo "Created new PR #${PR_NUMBER} for branch $PR_BRANCH" + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + fi + + - name: Merge PR with admin bypass + if: steps.commit.outputs.has_changes == 'true' && steps.automerge.outputs.enabled == 'true' + run: | + PR_NUMBER="${{ steps.create-or-update-pr.outputs.pr_number }}" + + if [[ -z "$PR_NUMBER" ]]; then + echo "❌ Could not find PR number to merge" + exit 1 + fi + + # The bot cannot go through a merge queue: the queue merges as + # GitHub, without the bot's ruleset bypass, so it would wait forever + # for a review that is not supposed to happen. Instead the bot + # merges directly — it bypasses the review rules but NOT the + # values-yaml-guard required check, so wait for it before merging. + # Check reads use the workflow token (GH_TOKEN wins over the + # step-level GITHUB_TOKEN env): the deployer app has no Checks:read + # permission, only the merge below needs the app identity. + # Poll until a required check run is registered before watching; + # gh pr checks --watch exits immediately with "no checks reported" + # when called right after gh pr create (github.com/cli/cli/issues/7401). + # stderr is left visible on purpose: discarding it previously + # disguised a token permission error as missing checks. + echo "Waiting for required checks to appear on PR #${PR_NUMBER}" + for attempt in $(seq 1 12); do + GH_TOKEN=${{ github.token }} gh pr checks "$PR_NUMBER" --required | grep -q . && break + echo "No required checks registered yet (attempt ${attempt}/12); waiting 5s..." + sleep 5 + done + + echo "Waiting for required checks to pass on PR #${PR_NUMBER}" + timeout 600 env GH_TOKEN=${{ github.token }} gh pr checks "$PR_NUMBER" --watch --required + + # A direct merge is exposed to the race where the base branch moves + # between check and merge — retry. --match-head-commit guards + # against a concurrent run pushing a new unchecked commit to the + # same branch after --watch returns. + COMMIT_OID="${{ steps.commit.outputs.commit_oid }}" + for attempt in 1 2 3; do + merge_output="$(gh pr merge "$PR_NUMBER" --merge --admin --match-head-commit "$COMMIT_OID" 2>&1)" && { + echo "$merge_output" + echo "✅ PR #${PR_NUMBER} merged" + exit 0 + } + + echo "$merge_output" >&2 + + if [[ "$merge_output" != *"Base branch was modified"* ]]; then + echo "❌ Merge failed on attempt ${attempt}" >&2 + exit 1 + fi + + echo "Base branch moved under PR #${PR_NUMBER}; retrying (attempt ${attempt}/3)" + sleep 10 + done + + echo "❌ Could not merge PR #${PR_NUMBER} after retries" >&2 + exit 1 + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} From d40c5c214681b2e9865f3e81c4b6549a646858f2 Mon Sep 17 00:00:00 2001 From: Romain Guichard Date: Wed, 8 Jul 2026 17:58:19 +0200 Subject: [PATCH 09/12] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Romain Guichard --- helm/charts/erpc/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/helm/charts/erpc/README.md b/helm/charts/erpc/README.md index 182999157..34f5dc5d6 100644 --- a/helm/charts/erpc/README.md +++ b/helm/charts/erpc/README.md @@ -8,8 +8,7 @@ For local testing and development: ```bash # Navigate to chart directory -cd charts/erpc/ - +cd helm/charts/erpc/ # Copy environment template and edit with your API keys cp .env.example .env # Edit .env with your actual API keys and AUTH_SECRET From fc6cb84500aff7574c8d9fbf3c4356f3e6916a81 Mon Sep 17 00:00:00 2001 From: Romain Guichard Date: Wed, 8 Jul 2026 18:00:40 +0200 Subject: [PATCH 10/12] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Romain Guichard --- helm/charts/erpc/templates/job-vault-db.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/helm/charts/erpc/templates/job-vault-db.yaml b/helm/charts/erpc/templates/job-vault-db.yaml index 8f45994b6..83eb51a47 100644 --- a/helm/charts/erpc/templates/job-vault-db.yaml +++ b/helm/charts/erpc/templates/job-vault-db.yaml @@ -75,12 +75,10 @@ spec: echo "Vault secrets file found. Creating Kubernetes Secret..." - DB_PASSWORD=$(cat /vault/secrets/secrets) - kubectl create secret generic erpc-db-secret \ --namespace={{ .Release.Namespace }} \ --from-literal=username=postgres \ - --from-literal=password="$DB_PASSWORD" \ + --from-file=password=/vault/secrets/secrets \ --dry-run=client -o yaml | kubectl apply -f - echo "Secret erpc-db-secret created successfully" From 98e2f74a2f6f8a908c80b1f5e1a161687ae43eb3 Mon Sep 17 00:00:00 2001 From: Romain Guichard Date: Wed, 8 Jul 2026 18:00:52 +0200 Subject: [PATCH 11/12] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Romain Guichard --- helm/charts/erpc/templates/pdb.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helm/charts/erpc/templates/pdb.yaml b/helm/charts/erpc/templates/pdb.yaml index 8f9a08f17..3e9a5fe2c 100644 --- a/helm/charts/erpc/templates/pdb.yaml +++ b/helm/charts/erpc/templates/pdb.yaml @@ -1,3 +1,4 @@ +{{- $replicas := ternary .Values.autoscaling.minReplicas .Values.replicaCount .Values.autoscaling.enabled -}} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -5,7 +6,7 @@ metadata: labels: {{- include "erpc.labels" . | nindent 4 }} spec: - minAvailable: {{ sub .Values.autoscaling.minReplicas 1 }} + minAvailable: {{ sub $replicas 1 }} selector: matchLabels: {{- include "erpc.selectorLabels" . | nindent 6 }} From 634fecd185fb336e587c258a16002bd8c5f2a2c0 Mon Sep 17 00:00:00 2001 From: rguichard Date: Thu, 9 Jul 2026 11:26:12 +0200 Subject: [PATCH 12/12] remove wrong gitignore lines --- .gitignore | 3 -- .../prd/erpc/templates/erpc-haproxy-hpa.yaml | 35 +++++++++++++++++++ .../erpc-processing-haproxy-hpa.yaml | 35 +++++++++++++++++++ .../templates/erpc-router-haproxy-hpa.yaml | 35 +++++++++++++++++++ 4 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 helm/environments/prd/erpc/templates/erpc-haproxy-hpa.yaml create mode 100644 helm/environments/prd/erpc/templates/erpc-processing-haproxy-hpa.yaml create mode 100644 helm/environments/prd/erpc/templates/erpc-router-haproxy-hpa.yaml diff --git a/.gitignore b/.gitignore index 26eb389af..6efcb8bae 100644 --- a/.gitignore +++ b/.gitignore @@ -11,9 +11,6 @@ vendor/ *.cover coverage.txt .DS_Store -erpc.yaml -erpc-*.yaml -erpc.ts dist/ sqd-simplify/ .generated-go-semantic-release-changelog.md diff --git a/helm/environments/prd/erpc/templates/erpc-haproxy-hpa.yaml b/helm/environments/prd/erpc/templates/erpc-haproxy-hpa.yaml new file mode 100644 index 000000000..afceea903 --- /dev/null +++ b/helm/environments/prd/erpc/templates/erpc-haproxy-hpa.yaml @@ -0,0 +1,35 @@ +{{- if (index .Values "erpc-haproxy-hpa" "enabled") }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: erpc-erpc-haproxy + labels: + app.kubernetes.io/name: haproxy + app.kubernetes.io/instance: erpc-erpc-haproxy +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: erpc-erpc-haproxy + minReplicas: {{ index .Values "erpc-haproxy-hpa" "minReplicas" }} + maxReplicas: {{ index .Values "erpc-haproxy-hpa" "maxReplicas" }} + metrics: + {{- if index .Values "erpc-haproxy-hpa" "targetCPUUtilizationPercentage" }} + - type: Pods + pods: + metric: + name: cpu_utilization_percentage + target: + type: AverageValue + averageValue: {{ index .Values "erpc-haproxy-hpa" "targetCPUUtilizationPercentage" }} + {{- end }} + {{- if index .Values "erpc-haproxy-hpa" "targetMemoryUtilizationPercentage" }} + - type: Pods + pods: + metric: + name: memory_utilization_percentage + target: + type: AverageValue + averageValue: {{ index .Values "erpc-haproxy-hpa" "targetMemoryUtilizationPercentage" }} + {{- end }} +{{- end }} diff --git a/helm/environments/prd/erpc/templates/erpc-processing-haproxy-hpa.yaml b/helm/environments/prd/erpc/templates/erpc-processing-haproxy-hpa.yaml new file mode 100644 index 000000000..0fa8b67e2 --- /dev/null +++ b/helm/environments/prd/erpc/templates/erpc-processing-haproxy-hpa.yaml @@ -0,0 +1,35 @@ +{{- if (index .Values "erpc-processing-haproxy-hpa" "enabled") }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: erpc-erpc-processing-haproxy + labels: + app.kubernetes.io/name: haproxy + app.kubernetes.io/instance: erpc-erpc-processing-haproxy +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: erpc-erpc-processing-haproxy + minReplicas: {{ index .Values "erpc-processing-haproxy-hpa" "minReplicas" }} + maxReplicas: {{ index .Values "erpc-processing-haproxy-hpa" "maxReplicas" }} + metrics: + {{- if index .Values "erpc-processing-haproxy-hpa" "targetCPUUtilizationPercentage" }} + - type: Pods + pods: + metric: + name: cpu_utilization_percentage + target: + type: AverageValue + averageValue: {{ index .Values "erpc-processing-haproxy-hpa" "targetCPUUtilizationPercentage" }} + {{- end }} + {{- if index .Values "erpc-processing-haproxy-hpa" "targetMemoryUtilizationPercentage" }} + - type: Pods + pods: + metric: + name: memory_utilization_percentage + target: + type: AverageValue + averageValue: {{ index .Values "erpc-processing-haproxy-hpa" "targetMemoryUtilizationPercentage" }} + {{- end }} +{{- end }} diff --git a/helm/environments/prd/erpc/templates/erpc-router-haproxy-hpa.yaml b/helm/environments/prd/erpc/templates/erpc-router-haproxy-hpa.yaml new file mode 100644 index 000000000..8fa9e42fc --- /dev/null +++ b/helm/environments/prd/erpc/templates/erpc-router-haproxy-hpa.yaml @@ -0,0 +1,35 @@ +{{- if (index .Values "erpc-router-haproxy-hpa" "enabled") }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: erpc-erpc-router-haproxy + labels: + app.kubernetes.io/name: haproxy + app.kubernetes.io/instance: erpc-erpc-router-haproxy +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: erpc-erpc-router-haproxy + minReplicas: {{ index .Values "erpc-router-haproxy-hpa" "minReplicas" }} + maxReplicas: {{ index .Values "erpc-router-haproxy-hpa" "maxReplicas" }} + metrics: + {{- if index .Values "erpc-router-haproxy-hpa" "targetCPUUtilizationPercentage" }} + - type: Pods + pods: + metric: + name: cpu_utilization_percentage + target: + type: AverageValue + averageValue: {{ index .Values "erpc-router-haproxy-hpa" "targetCPUUtilizationPercentage" }} + {{- end }} + {{- if index .Values "erpc-router-haproxy-hpa" "targetMemoryUtilizationPercentage" }} + - type: Pods + pods: + metric: + name: memory_utilization_percentage + target: + type: AverageValue + averageValue: {{ index .Values "erpc-router-haproxy-hpa" "targetMemoryUtilizationPercentage" }} + {{- end }} +{{- end }}