diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 000000000..1b3c6e201
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1,7 @@
+# Default ownership
+* @morpho-org/platform-engineers @0x666c6f
+
+# Helm charts migrated from morpho-infra-helm (2026-06-17)
+/helm/ @morpho-org/platform-engineers @morpho-org/api-engineers @0x666c6f
+/validate-helm-charts.sh @morpho-org/platform-engineers @0x666c6f
+/setup-hooks.sh @morpho-org/platform-engineers @0x666c6f
diff --git a/.github/workflows/helm-charts-validation.yaml b/.github/workflows/helm-charts-validation.yaml
new file mode 100644
index 000000000..49a93b59e
--- /dev/null
+++ b/.github/workflows/helm-charts-validation.yaml
@@ -0,0 +1,154 @@
+name: Helm Chart Validation
+
+on:
+ push:
+ branches: [morpho-main]
+ paths:
+ - 'helm/**'
+ pull_request:
+ branches: [morpho-main]
+ paths:
+ - 'helm/**'
+
+jobs:
+ # =============================================================================
+ # Job 1: Discover all charts to validate
+ # =============================================================================
+ discover-charts:
+ name: Discover Charts
+ runs-on: ubuntu-latest
+ outputs:
+ charts: ${{ steps.find.outputs.charts }}
+ chart-count: ${{ steps.find.outputs.count }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ persist-credentials: false
+
+ - name: Find all charts
+ id: find
+ run: |
+ # Find all Chart.yaml files and get their directories
+ charts=$(find helm/charts helm/environments -name Chart.yaml -type f -exec dirname {} \; | sort | jq -R -s -c 'split("\n") | map(select(length > 0))')
+ count=$(echo "$charts" | jq 'length')
+ echo "Found $count charts to validate"
+ echo "charts=$charts" >> $GITHUB_OUTPUT
+ echo "count=$count" >> $GITHUB_OUTPUT
+
+ # =============================================================================
+ # Job 2: Validate each chart in parallel
+ # =============================================================================
+ validate:
+ name: Validate
+ needs: discover-charts
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false # Continue validating other charts if one fails
+ matrix:
+ chart: ${{ fromJson(needs.discover-charts.outputs.charts) }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ persist-credentials: false
+
+ - name: Set up Helm
+ uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4
+ with:
+ version: v3.16.4
+
+ - name: Install kubeconform
+ run: |
+ KUBECONFORM_VERSION=v0.6.1
+ KUBECONFORM_SHA256=7f84c1e4109fb72f151da41bdffea05e1e37591f179de51adc048c83533ad215
+ curl -fsSLo /tmp/kubeconform.tar.gz "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz"
+ echo "${KUBECONFORM_SHA256} /tmp/kubeconform.tar.gz" | sha256sum -c -
+ tar xzf /tmp/kubeconform.tar.gz
+ sudo mv kubeconform /usr/local/bin/
+
+ - name: Add Helm repositories
+ run: |
+ helm repo add bitnami https://charts.bitnami.com/bitnami 2>/dev/null || true
+ helm repo add cnpg https://cloudnative-pg.github.io/charts 2>/dev/null || true
+ helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 2>/dev/null || true
+ helm repo add haproxytech https://haproxytech.github.io/helm-charts 2>/dev/null || true
+ helm repo add dandydeveloper https://dandydeveloper.github.io/charts 2>/dev/null || true
+ helm repo update
+
+ - name: Get chart info
+ id: chart-info
+ run: |
+ chart_path="${{ matrix.chart }}"
+ chart_name=$(basename "$chart_path")
+ echo "name=$chart_name" >> $GITHUB_OUTPUT
+ echo "path=$chart_path" >> $GITHUB_OUTPUT
+
+ # Check if chart should skip kubeconform validation
+ skip_validation="false"
+ case "$chart_name" in
+ *-db|*database*|*postgres*)
+ skip_validation="true"
+ ;;
+ esac
+ echo "skip-kubeconform=$skip_validation" >> $GITHUB_OUTPUT
+
+ - name: Helm lint
+ run: |
+ echo "::group::Helm lint ${{ steps.chart-info.outputs.name }}"
+ helm lint "${{ matrix.chart }}"
+ echo "::endgroup::"
+
+ - name: Update dependencies
+ run: |
+ cd "${{ matrix.chart }}"
+ if grep -q "dependencies:" Chart.yaml 2>/dev/null; then
+ echo "::group::Updating dependencies for ${{ steps.chart-info.outputs.name }}"
+ helm dependency update
+ echo "::endgroup::"
+ fi
+
+ - name: Kubeconform validation
+ if: steps.chart-info.outputs.skip-kubeconform != 'true'
+ run: |
+ CUSTOM_RESOURCES="PodMonitor,ServiceMonitor,AlertManager,Prometheus,PrometheusRule,Cluster,ScheduledBackup,ImageCatalog,IngressRoute,Certificate,CronJob,EventSource,EventBus,Sensor"
+
+ echo "::group::Kubeconform validation for ${{ steps.chart-info.outputs.name }}"
+ cd "${{ matrix.chart }}"
+ helm template . | kubeconform --ignore-missing-schemas --summary --skip "$CUSTOM_RESOURCES"
+ echo "::endgroup::"
+
+ - name: Skip notice
+ if: steps.chart-info.outputs.skip-kubeconform == 'true'
+ run: |
+ echo "::notice::Skipping kubeconform validation for ${{ steps.chart-info.outputs.name }} (database chart)"
+
+ # =============================================================================
+ # Job 3: Summary (ensures all validations completed)
+ # =============================================================================
+ validation-summary:
+ name: Validation Summary
+ needs: [discover-charts, validate]
+ runs-on: ubuntu-latest
+ if: always()
+ steps:
+ - name: Check validation results
+ run: |
+ if [ "${{ needs.discover-charts.result }}" == "failure" ]; then
+ echo "::error::Chart discovery failed"
+ exit 1
+ elif [ "${{ needs.discover-charts.result }}" == "cancelled" ]; then
+ echo "::warning::Chart discovery was cancelled"
+ exit 1
+ elif [ "${{ needs.validate.result }}" == "failure" ]; then
+ echo "::error::Some chart validations failed"
+ exit 1
+ elif [ "${{ needs.validate.result }}" == "cancelled" ]; then
+ echo "::warning::Validation was cancelled"
+ exit 1
+ elif [ "${{ needs.validate.result }}" == "skipped" ]; then
+ echo "::warning::Validation was skipped unexpectedly"
+ exit 1
+ else
+ echo "::notice::All ${{ needs.discover-charts.outputs.chart-count }} charts validated successfully"
+ fi
diff --git a/.github/workflows/only-values-yaml-guard.yml b/.github/workflows/only-values-yaml-guard.yml
new file mode 100644
index 000000000..2251b5a45
--- /dev/null
+++ b/.github/workflows/only-values-yaml-guard.yml
@@ -0,0 +1,63 @@
+# Required status check that scopes the morpho-infra-deployer bot's review
+# bypass to values.yaml-only changes: it is skipped unless the deployer bot
+# authored the PR or triggered the run, and fails only when such a PR touches
+# anything other than values.yaml.
+name: only-values-yaml-guard
+
+on:
+ pull_request:
+ branches: [morpho-main]
+ merge_group:
+
+permissions:
+ contents: read
+ pull-requests: read
+
+jobs:
+ guard:
+ name: values-yaml-guard
+ runs-on: ubuntu-latest
+ # Skipped for non-bot PRs: a skipped job SATISFIES required status checks,
+ # so human PRs merge normally without spinning up a runner. Covers the bot
+ # both as PR author and as pusher on someone else's PR, matching on the
+ # immutable user id (not the login) so an app rename cannot silently turn
+ # the guard off while the bot keeps its review-bypass.
+ if: github.event.pull_request.user.id == 187494245 || github.actor_id == 187494245 # morpho-infra-deployer[bot]
+ steps:
+ - name: Enforce values.yaml-only changes for the deployer bot
+ env:
+ GH_TOKEN: ${{ github.token }}
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ MERGE_GROUP_HEAD_REF: ${{ github.event.merge_group.head_ref }}
+ REPO: ${{ github.repository }}
+ run: |
+ # merge_group events carry no pull_request.number; extract it from
+ # the head ref (refs/heads/gh-readonly-queue//pr--).
+ if [[ -z "$PR_NUMBER" && -n "$MERGE_GROUP_HEAD_REF" ]]; then
+ PR_NUMBER=$(echo "$MERGE_GROUP_HEAD_REF" | grep -oE '/pr-[0-9]+' | grep -oE '[0-9]+')
+ fi
+
+ # Include previous_filename so a rename cannot smuggle a non-values
+ # path out of sight (the files API only reports the new path).
+ if ! mapfile -t CHANGED < <(gh api --paginate "repos/${REPO}/pulls/${PR_NUMBER}/files" --jq '.[] | .filename, (.previous_filename // empty)'); then
+ echo "❌ Failed to fetch changed files from the GitHub API."
+ exit 1
+ fi
+ echo "Files changed by the deployer bot:"
+ printf ' %s\n' "${CHANGED[@]}"
+
+ OFFENDING=()
+ for file in "${CHANGED[@]}"; do
+ if [[ "$file" != "values.yaml" && "$file" != *"/values.yaml" ]]; then
+ OFFENDING+=("$file")
+ fi
+ done
+
+ if (( ${#OFFENDING[@]} > 0 )); then
+ echo "❌ Deployer bot PR modifies files other than values.yaml:"
+ printf ' %s\n' "${OFFENDING[@]}"
+ echo "The bot may only bypass review for values.yaml-only changes."
+ exit 1
+ fi
+
+ echo "✅ Deployer bot PR only modifies values.yaml files."
diff --git a/.github/workflows/slack-pr-notification.yaml b/.github/workflows/slack-pr-notification.yaml
new file mode 100644
index 000000000..5a439d0fe
--- /dev/null
+++ b/.github/workflows/slack-pr-notification.yaml
@@ -0,0 +1,49 @@
+name: Slack PR Notification
+
+on:
+ pull_request:
+ types: [closed]
+ branches: [morpho-main]
+
+jobs:
+ notify-slack:
+ if: github.event.pull_request.merged == true && github.event.pull_request.user.login != 'github-actions[bot]' && github.event.pull_request.user.login != 'morpho-infra-deployer[bot]'
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Send Slack notification
+ uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+ with:
+ webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+ webhook-type: incoming-webhook
+ payload: |
+ {
+ "text": ":rocket: PR merged to Production",
+ "attachments": [
+ {
+ "color": "#ff6b6b",
+ "fallback": "PR #${{ github.event.pull_request.number }} merged: ${{ github.event.pull_request.title }}",
+ "title": ":white_check_mark: PR #${{ github.event.pull_request.number }}: ${{ github.event.pull_request.title }}",
+ "title_link": "${{ github.event.pull_request.html_url }}",
+ "fields": [
+ {
+ "title": ":busts_in_silhouette: Author",
+ "value": "${{ github.event.pull_request.user.login }}",
+ "short": true
+ },
+ {
+ "title": ":earth_americas: Environment",
+ "value": ":red_circle: Production (morpho-main)",
+ "short": true
+ },
+ {
+ "title": ":arrow_right: Branch",
+ "value": "`${{ github.event.pull_request.head.ref }}` → `${{ github.event.pull_request.base.ref }}`",
+ "short": false
+ }
+ ],
+ "footer": ":octopus: ${{ github.repository }}",
+ "footer_icon": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png"
+ }
+ ]
+ }
diff --git a/.github/workflows/update-image-tag.yaml b/.github/workflows/update-image-tag.yaml
new file mode 100644
index 000000000..9b4a0fdc2
--- /dev/null
+++ b/.github/workflows/update-image-tag.yaml
@@ -0,0 +1,291 @@
+name: Update chart image tag
+
+permissions: {}
+
+on:
+ workflow_dispatch:
+ inputs:
+ version_tag:
+ description: 'Release version to deploy (e.g., 0.1.3)'
+ required: true
+ type: string
+ release_url:
+ description: 'Release URL'
+ required: false
+ type: string
+
+jobs:
+ update-image-tag:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ pull-requests: write
+ checks: read
+ statuses: read
+ actions: read
+ env:
+ INPUT_VERSION_TAG: ${{ inputs.version_tag }}
+ INPUT_RELEASE_URL: ${{ inputs.release_url }}
+ ANCHOR_NAMES: 'erpcImageTag erpcValidatorImageTag'
+ steps:
+ - name: Validate version
+ run: |
+ if [[ ! "$INPUT_VERSION_TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+ echo "❌ Invalid version '$INPUT_VERSION_TAG'. Expected semver like 0.1.3"
+ exit 1
+ fi
+
+ - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ ref: morpho-main
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Determine auto-merge
+ id: automerge
+ run: |
+ # erpc only deploys to prd; auto-merge is opt-in via vars.ERPC_PRD_AUTOMERGE.
+ if [[ "${{ vars.ERPC_PRD_AUTOMERGE }}" == "true" ]]; then
+ echo "enabled=true" >> $GITHUB_OUTPUT
+ echo "✅ Auto-merge enabled"
+ else
+ echo "enabled=false" >> $GITHUB_OUTPUT
+ echo "⏸️ Auto-merge disabled — PR will require manual approval"
+ fi
+
+ - name: Generate GitHub App Token
+ id: generate-token
+ uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1
+ with:
+ app-id: ${{ secrets.INFRA_DEPLOYER_APP_ID }}
+ private-key: ${{ secrets.INFRA_DEPLOYER_SECRET_KEY }}
+
+ - name: Setup Git remote
+ run: |
+ git remote set-url origin https://x-access-token:${{ steps.generate-token.outputs.token }}@github.com/${{ github.repository }}.git
+
+ - name: Create feature branch
+ id: create-branch
+ run: |
+ PR_BRANCH="update-image-tag/erpc/prd"
+
+ if git ls-remote --heads origin "$PR_BRANCH" | grep -q "$PR_BRANCH"; then
+ echo "Branch $PR_BRANCH exists remotely, deleting it"
+ git push origin --delete "$PR_BRANCH" || true
+ fi
+
+ git checkout -b "$PR_BRANCH"
+ git push -f origin "$PR_BRANCH"
+ echo "pr_branch=$PR_BRANCH" >> $GITHUB_OUTPUT
+
+ - name: Update image tags across all values.yaml
+ id: update
+ run: |
+ NEW_VERSION="$INPUT_VERSION_TAG"
+
+ UPDATED=0
+ FILES_UPDATED=""
+
+ while IFS= read -r FILE; do
+ for ANCHOR in $ANCHOR_NAMES; do
+ if grep -qE "tag: &${ANCHOR} " "$FILE"; then
+ # Update the anchor value: 'tag: & ""' — the
+ # aliased (*) occurrences follow automatically.
+ sed -i.bak -E "s|(tag: &${ANCHOR} )\"[^\"]+\"|\1\"${NEW_VERSION}\"|" "$FILE"
+ rm -f "${FILE}.bak"
+ fi
+ done
+
+ if ! git diff --quiet "$FILE"; then
+ echo "✓ Updated: $FILE"
+ git --no-pager diff "$FILE" | head -10
+ UPDATED=$((UPDATED + 1))
+ FILES_UPDATED="${FILES_UPDATED}- \`${FILE}\`\n"
+ fi
+ done < <(find helm/environments/prd -name 'values.yaml' -type f | sort -u)
+
+ echo "Total files updated: $UPDATED"
+ echo "count=$UPDATED" >> $GITHUB_OUTPUT
+ {
+ echo "files_list<> $GITHUB_OUTPUT
+
+ - name: Create signed commit via API
+ id: commit
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ env:
+ PR_BRANCH: ${{ steps.create-branch.outputs.pr_branch }}
+ with:
+ github-token: ${{ steps.generate-token.outputs.token }}
+ script: |
+ const { execSync } = require("node:child_process");
+ const fs = require("node:fs");
+
+ const changed = execSync("git diff --name-only -- helm/environments", { encoding: "utf8" })
+ .split("\n")
+ .filter(Boolean);
+
+ if (changed.length === 0) {
+ core.setOutput("has_changes", "false");
+ core.info("No changes to commit — image tags already up to date");
+ return;
+ }
+
+ const version = process.env.INPUT_VERSION_TAG;
+ const url = process.env.INPUT_RELEASE_URL || "n/a";
+
+ const additions = changed.map((path) => ({
+ path,
+ contents: fs.readFileSync(path).toString("base64"),
+ }));
+
+ const expectedHeadOid = execSync("git rev-parse HEAD", { encoding: "utf8" }).trim();
+
+ const { createCommitOnBranch } = await github.graphql(
+ `mutation ($input: CreateCommitOnBranchInput!) {
+ createCommitOnBranch(input: $input) {
+ commit { oid }
+ }
+ }`,
+ {
+ input: {
+ branch: {
+ repositoryNameWithOwner: `${context.repo.owner}/${context.repo.repo}`,
+ branchName: process.env.PR_BRANCH,
+ },
+ message: {
+ headline: `[erpc] 🤖 Update image tags to ${version} in production`,
+ body: `Updates the erpc chart deployments to use the latest image.\n\nVersion: ${version}\nRelease URL: ${url}`,
+ },
+ fileChanges: { additions },
+ expectedHeadOid,
+ },
+ },
+ );
+
+ core.info(`Created signed commit ${createCommitOnBranch.commit.oid}`);
+ core.setOutput("has_changes", "true");
+ core.setOutput("commit_oid", createCommitOnBranch.commit.oid);
+
+ - name: Create or update PR
+ id: create-or-update-pr
+ if: steps.commit.outputs.has_changes == 'true'
+ env:
+ FILES_LIST: ${{ steps.update.outputs.files_list }}
+ GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}
+ run: |
+ PR_BRANCH="${{ steps.create-branch.outputs.pr_branch }}"
+
+ if [[ "${{ steps.automerge.outputs.enabled }}" == "true" ]]; then
+ AUTO_MERGE_STATUS="✅ **Auto-merged** - This PR is automatically merged once its required checks pass."
+ else
+ AUTO_MERGE_STATUS="⏸️ **Manual approval required** - This PR requires manual review and approval before merging."
+ fi
+
+ PR_TITLE="chore(helm): update erpc image tags in production"
+ PR_BODY="## 🤖 Automated Image Update
+
+ This PR updates all erpc chart deployments in **production** to the latest image.
+
+ ### Files Updated (${{ steps.update.outputs.count }})
+ ${FILES_LIST}
+
+ ### Source Information
+ - Version: ${INPUT_VERSION_TAG}
+ - Release URL: ${INPUT_RELEASE_URL}
+ - Anchors: \`${ANCHOR_NAMES}\`
+
+ ### Merge Status
+ ${AUTO_MERGE_STATUS}"
+
+ # The head branch was deleted and recreated earlier in this run, which
+ # auto-closes any previous PR. gh pr list relies on the eventually
+ # consistent search index and can still return that just-closed PR,
+ # so confirm the state with a strongly consistent direct lookup
+ # before editing instead of creating.
+ PR_NUMBER=$(gh pr list --head "$PR_BRANCH" --json number --jq '.[0].number // empty')
+ if [[ -n "$PR_NUMBER" ]] && [[ "$(gh pr view "$PR_NUMBER" --json state --jq '.state')" == "OPEN" ]]; then
+ echo "PR #$PR_NUMBER already exists for branch $PR_BRANCH, updating it"
+ gh pr edit "$PR_NUMBER" --base "morpho-main" --title "$PR_TITLE" --body "$PR_BODY"
+ echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT
+ else
+ PR_CREATE_ARGS=(
+ --title "$PR_TITLE"
+ --body "$PR_BODY"
+ --base "morpho-main"
+ --head "$PR_BRANCH"
+ )
+
+ if [[ "${{ steps.automerge.outputs.enabled }}" != "true" ]]; then
+ PR_CREATE_ARGS+=(--draft)
+ fi
+
+ PR_URL=$(gh pr create "${PR_CREATE_ARGS[@]}")
+
+ PR_NUMBER=$(echo "$PR_URL" | grep -oE '[0-9]+$')
+ echo "Created new PR #${PR_NUMBER} for branch $PR_BRANCH"
+ echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Merge PR with admin bypass
+ if: steps.commit.outputs.has_changes == 'true' && steps.automerge.outputs.enabled == 'true'
+ run: |
+ PR_NUMBER="${{ steps.create-or-update-pr.outputs.pr_number }}"
+
+ if [[ -z "$PR_NUMBER" ]]; then
+ echo "❌ Could not find PR number to merge"
+ exit 1
+ fi
+
+ # The bot cannot go through a merge queue: the queue merges as
+ # GitHub, without the bot's ruleset bypass, so it would wait forever
+ # for a review that is not supposed to happen. Instead the bot
+ # merges directly — it bypasses the review rules but NOT the
+ # values-yaml-guard required check, so wait for it before merging.
+ # Check reads use the workflow token (GH_TOKEN wins over the
+ # step-level GITHUB_TOKEN env): the deployer app has no Checks:read
+ # permission, only the merge below needs the app identity.
+ # Poll until a required check run is registered before watching;
+ # gh pr checks --watch exits immediately with "no checks reported"
+ # when called right after gh pr create (github.com/cli/cli/issues/7401).
+ # stderr is left visible on purpose: discarding it previously
+ # disguised a token permission error as missing checks.
+ echo "Waiting for required checks to appear on PR #${PR_NUMBER}"
+ for attempt in $(seq 1 12); do
+ GH_TOKEN=${{ github.token }} gh pr checks "$PR_NUMBER" --required | grep -q . && break
+ echo "No required checks registered yet (attempt ${attempt}/12); waiting 5s..."
+ sleep 5
+ done
+
+ echo "Waiting for required checks to pass on PR #${PR_NUMBER}"
+ timeout 600 env GH_TOKEN=${{ github.token }} gh pr checks "$PR_NUMBER" --watch --required
+
+ # A direct merge is exposed to the race where the base branch moves
+ # between check and merge — retry. --match-head-commit guards
+ # against a concurrent run pushing a new unchecked commit to the
+ # same branch after --watch returns.
+ COMMIT_OID="${{ steps.commit.outputs.commit_oid }}"
+ for attempt in 1 2 3; do
+ merge_output="$(gh pr merge "$PR_NUMBER" --merge --admin --match-head-commit "$COMMIT_OID" 2>&1)" && {
+ echo "$merge_output"
+ echo "✅ PR #${PR_NUMBER} merged"
+ exit 0
+ }
+
+ echo "$merge_output" >&2
+
+ if [[ "$merge_output" != *"Base branch was modified"* ]]; then
+ echo "❌ Merge failed on attempt ${attempt}" >&2
+ exit 1
+ fi
+
+ echo "Base branch moved under PR #${PR_NUMBER}; retrying (attempt ${attempt}/3)"
+ sleep 10
+ done
+
+ echo "❌ Could not merge PR #${PR_NUMBER} after retries" >&2
+ exit 1
+ env:
+ GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}
diff --git a/.github/workflows/wiz-iac-scan.yml b/.github/workflows/wiz-iac-scan.yml
new file mode 100644
index 000000000..6e2df7148
--- /dev/null
+++ b/.github/workflows/wiz-iac-scan.yml
@@ -0,0 +1,37 @@
+name: Wiz IaC Scan
+
+on:
+ pull_request:
+ paths:
+ - 'helm/**'
+
+permissions: {}
+
+jobs:
+ wiz-iac-scan:
+ name: Wiz IaC Scan
+ if: github.event.pull_request.head.repo.full_name == github.repository
+ runs-on: arc-morpho-infra-dev
+ permissions:
+ contents: read
+ env:
+ WIZCLI_VERSION: "1.45.0"
+ WIZCLI_SHA256: "3f25af9edd63dbd7ecd058b71912f8415467be3f0fcb2b28c6914ebc454d368e"
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+ with:
+ persist-credentials: false
+
+ - name: Download Wiz CLI
+ run: |
+ curl -fsSLo wizcli "https://downloads.wiz.io/v1/wizcli/${WIZCLI_VERSION}/wizcli-linux-amd64"
+ echo "${WIZCLI_SHA256} wizcli" | sha256sum -c -
+ chmod +x wizcli
+
+ - name: Run Wiz CLI IaC scan
+ env:
+ WIZ_CLIENT_ID: ${{ secrets.WIZ_CLIENT_ID }}
+ WIZ_CLIENT_SECRET: ${{ secrets.WIZ_CLIENT_SECRET }}
+ run: ./wizcli scan dir helm/
diff --git a/.gitignore b/.gitignore
index 26eb389af..6efcb8bae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,9 +11,6 @@ vendor/
*.cover
coverage.txt
.DS_Store
-erpc.yaml
-erpc-*.yaml
-erpc.ts
dist/
sqd-simplify/
.generated-go-semantic-release-changelog.md
diff --git a/helm/charts/.gitkeep b/helm/charts/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/helm/charts/erpc/.env.example b/helm/charts/erpc/.env.example
new file mode 100644
index 000000000..c6389d481
--- /dev/null
+++ b/helm/charts/erpc/.env.example
@@ -0,0 +1,23 @@
+# eRPC Environment Configuration
+# Copy this file to .env and fill in your actual API keys
+
+# Provider API Keys
+ALCHEMY_API_KEY=your-alchemy-api-key-here
+DRPC_API_KEY=your-drpc-api-key-here
+TENDERLY_API_KEY=your-tenderly-api-key-here
+TENDERLY_PROJECT=your-tenderly-project-slug
+ERPC_API_KEY=your-erpc-api-key-here
+
+# Authentication
+LABELS_SERVICE_API_KEY=labels-service-api-key-here
+SQD_INDEXERS_API_KEY=sqd-indexers-api-key-here
+BLUE_API_API_KEY=blue-api-key-here
+
+# Optional Configuration
+LOG_LEVEL=info
+
+# Instructions:
+# 1. Copy this file: cp .env.example .env
+# 2. Edit .env with your actual API keys
+# 3. Run: docker-compose up -d
+# 4. Test: ./test-local.sh
\ No newline at end of file
diff --git a/helm/charts/erpc/Chart.yaml b/helm/charts/erpc/Chart.yaml
new file mode 100644
index 000000000..ecc04b052
--- /dev/null
+++ b/helm/charts/erpc/Chart.yaml
@@ -0,0 +1,24 @@
+apiVersion: v2
+name: erpc
+description: eRPC fault-tolerant EVM RPC proxy
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.35
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "latest"
diff --git a/helm/charts/erpc/README.md b/helm/charts/erpc/README.md
new file mode 100644
index 000000000..34f5dc5d6
--- /dev/null
+++ b/helm/charts/erpc/README.md
@@ -0,0 +1,60 @@
+# eRPC Helm Chart
+
+This Helm chart deploys eRPC as a fault-tolerant EVM RPC proxy service within the Morpho infrastructure.
+
+## Local Development with Docker Compose
+
+For local testing and development:
+
+```bash
+# Navigate to chart directory
+cd helm/charts/erpc/
+# Copy environment template and edit with your API keys
+cp .env.example .env
+# Edit .env with your actual API keys and AUTH_SECRET
+
+# Start services (Redis + eRPC)
+docker-compose up -d
+
+# Run tests
+./test-local.sh
+
+# Stop services
+docker-compose down
+```
+
+
+## Usage
+
+When authentication is enabled, requests must include the secret as a query parameter:
+
+```bash
+curl --location 'http://localhost:4000/cache/evm/1?secret=YOUR-AUTH-SECRET' \
+ --header 'Content-Type: application/json' \
+ --data '{"method":"eth_blockNumber","params":[],"id":1,"jsonrpc":"2.0"}'
+```
+
+## Vault Config
+
+The eRPC chart reads non-secret config from `vault.configPath` and secret values
+from `vault.secretsPath`. Keep `vault.configPath` as a config template. Static
+secret placeholders use `__SECRET___`; the API-key auth list can use the
+`__SECRET_API_KEY_STRATEGIES__` marker to generate one secret auth strategy for
+every `API_KEY_*=` entry in `vault.secretsPath`. Generated auth IDs are
+derived from the key name after `API_KEY_`, lowercased with `_` converted to `-`;
+trailing hash rotation suffixes like `_9E41B694` are ignored so rotating a value
+does not change the eRPC secret ID. Optional per-key metadata entries can tune
+generated strategies without creating new API keys: `API_KEY_FOO__ID` pins the
+eRPC secret ID, `API_KEY_FOO__ORDER` pins list order, `API_KEY_FOO__ALLOW_METHODS`
+is a comma-separated method allowlist, and `API_KEY_FOO__RATE_LIMIT_BUDGET` sets
+`rateLimitBudget`.
+
+Vault secret keys must match `[A-Za-z_][A-Za-z0-9_]*`. Secret values are parsed
+from a line-oriented `KEY=VALUE` file, so keep them single-line. Placeholders in
+quoted YAML scalars are YAML-escaped by the renderer; placeholders embedded in
+unquoted YAML scalars, such as URL path fragments, must have YAML-plain-safe
+values.
+
+The Vault config job renders placeholders into a temporary `erpc.yaml`, validates
+that rendered config with `erpc validate`, then stores only the rendered file in
+the runtime Kubernetes Secret.
diff --git a/helm/charts/erpc/docker-compose.yml b/helm/charts/erpc/docker-compose.yml
new file mode 100644
index 000000000..09decb3af
--- /dev/null
+++ b/helm/charts/erpc/docker-compose.yml
@@ -0,0 +1,14 @@
+services:
+ erpc:
+ image: ghcr.io/erpc/erpc:latest
+ ports:
+ - "4000:4000" # HTTP RPC port
+ - "4001:4001" # Metrics port
+ volumes:
+ - ./config/erpc.yaml:/erpc.yaml:ro
+ env_file:
+ - .env
+ environment:
+ # DynamoDB configuration uses AWS credentials from environment or IAM
+ AWS_REGION: "eu-west-3"
+ restart: unless-stopped
\ No newline at end of file
diff --git a/helm/charts/erpc/templates/_helpers.tpl b/helm/charts/erpc/templates/_helpers.tpl
new file mode 100644
index 000000000..2b72a303d
--- /dev/null
+++ b/helm/charts/erpc/templates/_helpers.tpl
@@ -0,0 +1,87 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "erpc.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "erpc.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "erpc.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "erpc.labels" -}}
+helm.sh/chart: {{ include "erpc.chart" . }}
+{{ include "erpc.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "erpc.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "erpc.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Vault creator ServiceAccount name.
+*/}}
+{{- define "erpc.vaultCreatorServiceAccountName" -}}
+{{- $vaultCreator := .Values.serviceAccount.vaultCreator | default dict -}}
+{{- $vaultCreator.name | default .Values.serviceAccount.name | default "erpc" }}
+{{- end }}
+
+{{/*
+Runtime ServiceAccount name.
+*/}}
+{{- define "erpc.serviceAccountName" -}}
+{{- if .Values.serviceAccount.runtimeName -}}
+{{- .Values.serviceAccount.runtimeName -}}
+{{- else if eq (toString .Values.serviceAccount.create) "false" -}}
+{{- include "erpc.vaultCreatorServiceAccountName" . -}}
+{{- else -}}
+{{- printf "%s-runtime" (include "erpc.vaultCreatorServiceAccountName" .) -}}
+{{- end -}}
+{{- end }}
+
+{{/*
+Kubernetes Secrets managed by Vault creator jobs.
+*/}}
+{{- define "erpc.vaultSecretResourceNames" -}}
+- "erpc-db-secret"
+- {{ printf "%s-vault-config" (include "erpc.fullname" .) | quote }}
+{{- end }}
+
+{{/*
+Common environment variables for erpc
+*/}}
+{{- define "erpc.commonEnv" -}}
+# All env vars now come from vault-secrets via envFrom
+{{- end -}}
diff --git a/helm/charts/erpc/templates/deployment.yaml b/helm/charts/erpc/templates/deployment.yaml
new file mode 100644
index 000000000..0044065b3
--- /dev/null
+++ b/helm/charts/erpc/templates/deployment.yaml
@@ -0,0 +1,139 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "erpc.fullname" . }}
+ annotations:
+ reloader.stakater.com/auto: "true"
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+spec:
+ {{- if not .Values.autoscaling.enabled }}
+ replicas: {{ .Values.replicaCount }}
+ {{- end }}
+ strategy:
+ rollingUpdate:
+ # Let the new ReplicaSet surge relative to the current HPA target,
+ # not just minReplicas, so rollouts can warm a full replacement pool.
+ maxSurge: 100%
+ maxUnavailable: 0
+ type: RollingUpdate
+ selector:
+ matchLabels:
+ {{- include "erpc.selectorLabels" . | nindent 6 }}
+ template:
+ metadata:
+ labels:
+ {{- include "erpc.selectorLabels" . | nindent 8 }}
+ # Environment variables loaded from vault secrets via pre-install job
+ spec:
+ terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 45 }}
+ {{- with .Values.imagePullSecrets }}
+ imagePullSecrets:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ serviceAccountName: {{ include "erpc.serviceAccountName" . }}
+ securityContext:
+ {{- if .Values.securityHardening.enabled }}
+ {{- toYaml .Values.securityHardening.podSecurityContext | nindent 8 }}
+ {{- else }}
+ {{- toYaml .Values.podSecurityContext | nindent 8 }}
+ {{- end }}
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchLabels:
+ {{- include "erpc.selectorLabels" . | nindent 14 }}
+ - maxSkew: 1
+ topologyKey: topology.kubernetes.io/zone
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchLabels:
+ {{- include "erpc.selectorLabels" . | nindent 14 }}
+ containers:
+ - name: {{ .Chart.Name }}
+ securityContext:
+ {{- if .Values.securityHardening.enabled }}
+ {{- toYaml .Values.securityHardening.containerSecurityContext | nindent 12 }}
+ {{- else }}
+ {{- toYaml .Values.securityContext | nindent 12 }}
+ {{- end }}
+ image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ ports:
+ - name: http
+ containerPort: {{ .Values.config.server.httpPort }}
+ protocol: TCP
+ - name: metrics
+ containerPort: {{ .Values.config.server.metricsPort }}
+ protocol: TCP
+ env:
+ {{- with .Values.extraEnvVars }}
+ {{- toYaml . | nindent 12 }}
+ {{- end }}
+ volumeMounts:
+ - name: config
+ mountPath: /erpc.yaml
+ subPath: erpc.yaml
+ readOnly: true
+ {{- if .Values.securityHardening.enabled }}
+ - name: tmp
+ mountPath: /tmp
+ {{- end }}
+ startupProbe:
+ httpGet:
+ path: {{ .Values.startupProbe.path | default "/healthcheck" }}
+ port: http
+ initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds | default 10 }}
+ periodSeconds: {{ .Values.startupProbe.periodSeconds | default 5 }}
+ timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds | default 5 }}
+ failureThreshold: {{ .Values.startupProbe.failureThreshold | default 6 }}
+ livenessProbe:
+ tcpSocket:
+ port: http
+ {{- if .Values.livenessProbe.initialDelaySeconds }}
+ initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+ {{- end }}
+ periodSeconds: {{ .Values.livenessProbe.periodSeconds | default 10 }}
+ timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds | default 5 }}
+ successThreshold: 1
+ failureThreshold: {{ .Values.livenessProbe.failureThreshold | default 3 }}
+ readinessProbe:
+ httpGet:
+ path: {{ .Values.readinessProbe.path | default "/healthcheck" }}
+ port: http
+ {{- if .Values.readinessProbe.initialDelaySeconds }}
+ initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+ {{- end }}
+ periodSeconds: {{ .Values.readinessProbe.periodSeconds | default 5 }}
+ timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds | default 30 }}
+ successThreshold: 1
+ failureThreshold: {{ .Values.readinessProbe.failureThreshold | default 2 }}
+ resources:
+ {{- toYaml .Values.resources | nindent 12 }}
+ {{- if .Values.lifecycle.enabled }}
+ lifecycle:
+ {{- omit .Values.lifecycle "enabled" | toYaml | nindent 12 }}
+ {{- end }}
+ volumes:
+ - name: config
+ secret:
+ secretName: {{ include "erpc.fullname" . }}-vault-config
+ optional: true
+ {{- if .Values.securityHardening.enabled }}
+ - name: tmp
+ emptyDir: {}
+ {{- end }}
+ {{- with .Values.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.affinity }}
+ affinity:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
diff --git a/helm/charts/erpc/templates/hpa.yaml b/helm/charts/erpc/templates/hpa.yaml
new file mode 100644
index 000000000..b65a1cfb8
--- /dev/null
+++ b/helm/charts/erpc/templates/hpa.yaml
@@ -0,0 +1,38 @@
+{{- if .Values.autoscaling.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ name: {{ include "erpc.fullname" . }}
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: {{ include "erpc.fullname" . }}
+ minReplicas: {{ .Values.autoscaling.minReplicas }}
+ maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+ metrics:
+ {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
+ - type: Pods
+ pods:
+ metric:
+ name: cpu_utilization_percentage
+ target:
+ type: AverageValue
+ averageValue: "{{ .Values.autoscaling.targetCPUUtilizationPercentage }}"
+ {{- end }}
+ {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
+ - type: Pods
+ pods:
+ metric:
+ name: memory_utilization_percentage
+ target:
+ type: AverageValue
+ averageValue: "{{ .Values.autoscaling.targetMemoryUtilizationPercentage }}"
+ {{- end }}
+ {{- with .Values.autoscaling.behavior }}
+ behavior:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+{{- end }}
diff --git a/helm/charts/erpc/templates/image-catalog-pg18.yaml b/helm/charts/erpc/templates/image-catalog-pg18.yaml
new file mode 100644
index 000000000..d265b7149
--- /dev/null
+++ b/helm/charts/erpc/templates/image-catalog-pg18.yaml
@@ -0,0 +1,12 @@
+{{- if ne (toString .Values.postgresql.enabled) "false" }}
+apiVersion: postgresql.cnpg.io/v1
+kind: ImageCatalog
+metadata:
+ name: erpc-postgresql-native-18
+ annotations:
+ argocd.argoproj.io/sync-wave: "-3"
+spec:
+ images:
+ - major: 18
+ image: "ghcr.io/cloudnative-pg/postgresql:18.0-system-trixie"
+{{- end }}
diff --git a/helm/charts/erpc/templates/job-vault-config.yaml b/helm/charts/erpc/templates/job-vault-config.yaml
new file mode 100644
index 000000000..bf1d58ada
--- /dev/null
+++ b/helm/charts/erpc/templates/job-vault-config.yaml
@@ -0,0 +1,444 @@
+{{- $validationImage := .Values.vault.validationImage | default dict }}
+{{- if and $validationImage.enabled (or (not $validationImage.repository) (not $validationImage.tag)) }}
+{{- fail "vault.validationImage.repository and vault.validationImage.tag are required when vault.validationImage.enabled=true" }}
+{{- end }}
+{{- $useDedicatedValidator := $validationImage.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ include "erpc.fullname" . }}-vault-config-creator
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+ annotations:
+ "helm.sh/hook": pre-install,pre-upgrade
+ "helm.sh/hook-weight": "1"
+ "helm.sh/hook-delete-policy": hook-succeeded
+ argocd.argoproj.io/sync-wave: "-3"
+ argocd.argoproj.io/sync-options: Force=true,Replace=true
+spec:
+ template:
+ metadata:
+ labels:
+ {{- include "erpc.selectorLabels" . | nindent 8 }}
+ job: vault-config-creator
+ annotations:
+ # Vault Agent annotations for config injection (on pod template)
+ vault.hashicorp.com/agent-inject: "true"
+ vault.hashicorp.com/role: {{ include "erpc.vaultCreatorServiceAccountName" . | quote }}
+ vault.hashicorp.com/agent-inject-secret-erpc-config: {{ .Values.vault.configPath | default "secret/data/erpc/config" | quote }}
+ vault.hashicorp.com/agent-inject-secret-erpc-secrets: {{ .Values.vault.secretsPath | default "secret/data/erpc/secrets" | quote }}
+ vault.hashicorp.com/agent-inject-template-erpc-config: |
+ {{`{{- with secret "`}}{{ .Values.vault.configPath | default "secret/data/erpc/config" }}{{`" -}}
+ {{- .Data.data.config -}}
+ {{- end }}`}}
+ vault.hashicorp.com/agent-inject-template-erpc-secrets: |
+ {{`{{- with secret "`}}{{ .Values.vault.secretsPath | default "secret/data/erpc/secrets" }}{{`" -}}
+ {{- range $key, $value := .Data.data -}}
+ {{ printf "%s=%s\n" $key $value -}}
+ {{- end -}}
+ {{- end }}`}}
+ vault.hashicorp.com/agent-init-first: "true"
+ vault.hashicorp.com/agent-pre-populate-only: "true"
+ spec:
+ serviceAccountName: {{ include "erpc.vaultCreatorServiceAccountName" . }}
+ automountServiceAccountToken: true
+ {{- if .Values.securityHardening.enabled }}
+ securityContext:
+ {{- toYaml .Values.securityHardening.podSecurityContext | nindent 8 }}
+ {{- end }}
+ restartPolicy: OnFailure
+ volumes:
+ - name: rendered-config
+ emptyDir: {}
+ {{- if .Values.securityHardening.enabled }}
+ - name: tmp
+ emptyDir: {}
+ {{- end }}
+ initContainers:
+ - name: config-renderer
+ {{- if .Values.securityHardening.enabled }}
+ securityContext:
+ {{- toYaml .Values.securityHardening.containerSecurityContext | nindent 12 }}
+ {{- end }}
+ image: "{{ .Values.vault.jobImage.repository }}:{{ .Values.vault.jobImage.tag }}"
+ imagePullPolicy: {{ .Values.vault.jobImage.pullPolicy }}
+ resources:
+ {{- toYaml .Values.vault.jobResources | nindent 12 }}
+ {{- if .Values.securityHardening.enabled }}
+ env:
+ - name: HOME
+ value: /tmp
+ {{- end }}
+ volumeMounts:
+ - name: rendered-config
+ mountPath: /work
+ {{- if .Values.securityHardening.enabled }}
+ - name: tmp
+ mountPath: /tmp
+ {{- end }}
+ command:
+ - sh
+ - -c
+ - |
+ set -eu
+ if (set -o pipefail) 2>/dev/null; then
+ set -o pipefail
+ fi
+
+ echo "Rendering ERPC configuration from vault config and secrets..."
+
+ timeout=60
+ count=0
+ while { [ ! -f /vault/secrets/erpc-config ] || [ ! -f /vault/secrets/erpc-secrets ]; } && [ $count -lt $timeout ]; do
+ echo "Waiting for vault config and secrets files... ($count/$timeout)"
+ sleep 2
+ count=$((count + 2))
+ done
+
+ if [ ! -f /vault/secrets/erpc-config ]; then
+ echo "ERROR: Vault config file not found after $timeout seconds"
+ exit 1
+ fi
+
+ if [ ! -f /vault/secrets/erpc-secrets ]; then
+ echo "ERROR: Vault secrets file not found after $timeout seconds"
+ exit 1
+ fi
+
+ tmp_rendered_config=/tmp/erpc-rendered-config
+ rendered_config=/work/erpc.yaml
+ trap 'rm -f "$tmp_rendered_config"' EXIT
+
+ awk -v secrets_file="/vault/secrets/erpc-secrets" '
+ function yaml_quote(value, quoted) {
+ quoted = value
+ gsub(/\\/, "\\\\", quoted)
+ gsub(/"/, "\\\"", quoted)
+ gsub(/\r/, "\\r", quoted)
+ gsub(/\n/, "\\n", quoted)
+ gsub(/\t/, "\\t", quoted)
+ return "\"" quoted "\""
+ }
+
+ function metadata_key(key, suffix) {
+ return key "__" suffix
+ }
+
+ function api_key_base(key, suffix) {
+ suffix = substr(key, length(key) - 8)
+ if (suffix ~ /^_[0-9A-Fa-f]{8}$/) {
+ return substr(key, 1, length(key) - 9)
+ }
+ return key
+ }
+
+ function metadata_lookup_key(key, suffix, exact_key, base_key) {
+ exact_key = metadata_key(key, suffix)
+ if (exact_key in secrets) {
+ return exact_key
+ }
+
+ base_key = metadata_key(api_key_base(key), suffix)
+ if (base_key in secrets) {
+ return base_key
+ }
+
+ return ""
+ }
+
+ function api_key_id(key, id, id_key) {
+ id_key = metadata_lookup_key(key, "ID")
+ if (id_key != "") {
+ return secrets[id_key]
+ }
+
+ id = substr(api_key_base(key), 9)
+ id = tolower(id)
+ gsub(/_+/, "-", id)
+ return id
+ }
+
+ function trim(value) {
+ sub(/^[[:space:]]+/, "", value)
+ sub(/[[:space:]]+$/, "", value)
+ return value
+ }
+
+ function emit_method_list(indent, methods, parts, count, i, method) {
+ count = split(methods, parts, ",")
+ for (i = 1; i <= count; i++) {
+ method = trim(parts[i])
+ if (method != "") {
+ print indent " - " yaml_quote(method)
+ }
+ }
+ }
+
+ function api_key_sort_value(key, order_key) {
+ order_key = metadata_lookup_key(key, "ORDER")
+ if (order_key != "" && secrets[order_key] ~ /^[0-9]+$/) {
+ return sprintf("%09d:%s", secrets[order_key] + 0, key)
+ }
+ return "999999999:" key
+ }
+
+ function sort_api_keys(i, j, tmp, left, right) {
+ if (api_key_order_sorted) {
+ return
+ }
+
+ for (i = 1; i <= api_key_count; i++) {
+ for (j = i + 1; j <= api_key_count; j++) {
+ left = api_key_sort_value(api_key_order[i])
+ right = api_key_sort_value(api_key_order[j])
+ if (right < left) {
+ tmp = api_key_order[i]
+ api_key_order[i] = api_key_order[j]
+ api_key_order[j] = tmp
+ }
+ }
+ }
+ api_key_order_sorted = 1
+ }
+
+ function emit_api_key_strategies(indent, i, key, allow_key, budget_key) {
+ if (api_key_count == 0) {
+ print "ERROR: __SECRET_API_KEY_STRATEGIES__ used but no API_KEY_* entries exist in Vault secrets" > "/dev/stderr"
+ exit 1
+ }
+
+ sort_api_keys()
+ for (i = 1; i <= api_key_count; i++) {
+ key = api_key_order[i]
+ allow_key = metadata_lookup_key(key, "ALLOW_METHODS")
+ budget_key = metadata_lookup_key(key, "RATE_LIMIT_BUDGET")
+
+ print indent "- type: secret"
+ if (budget_key != "") {
+ print indent " rateLimitBudget: " yaml_quote(secrets[budget_key])
+ }
+ if (allow_key != "") {
+ print indent " ignoreMethods:"
+ print indent " - " yaml_quote("*")
+ }
+ print indent " secret:"
+ print indent " id: " yaml_quote(api_key_id(key))
+ print indent " value: " yaml_quote(secrets[key])
+ if (allow_key != "") {
+ print indent " allowMethods:"
+ emit_method_list(indent, secrets[allow_key])
+ }
+ }
+ }
+
+ BEGIN {
+ missing_count = 0
+ invalid_count = 0
+ api_key_count = 0
+ api_key_order_sorted = 0
+ while ((getline line < secrets_file) > 0) {
+ if (line == "") {
+ continue
+ }
+
+ delimiter = index(line, "=")
+ if (delimiter == 0) {
+ print "ERROR: Invalid Vault secret entry without = delimiter" > "/dev/stderr"
+ invalid_count++
+ continue
+ }
+
+ key = substr(line, 1, delimiter - 1)
+ value = substr(line, delimiter + 1)
+
+ if (key !~ /^[A-Za-z_][A-Za-z0-9_]*$/) {
+ print "ERROR: Invalid Vault secret key " key ". Use [A-Za-z_][A-Za-z0-9_]*." > "/dev/stderr"
+ invalid_count++
+ continue
+ }
+
+ secrets[key] = value
+ if (key ~ /^API_KEY_/ && key !~ /__(ALLOW_METHODS|RATE_LIMIT_BUDGET|ID|ORDER)$/) {
+ api_key_order[++api_key_count] = key
+ }
+ }
+ close(secrets_file)
+
+ if (invalid_count > 0) {
+ exit 1
+ }
+ }
+ {
+ while (match($0, /"__SECRET_[A-Za-z_][A-Za-z0-9_]*__"/)) {
+ token = substr($0, RSTART + 1, RLENGTH - 2)
+ key = substr(token, 10, length(token) - 11)
+
+ if (key in secrets) {
+ repl = yaml_quote(secrets[key])
+ } else {
+ repl = yaml_quote("")
+ if (!(token in missing)) {
+ missing[token] = 1
+ missing_order[++missing_count] = token
+ }
+ }
+
+ $0 = substr($0, 1, RSTART - 1) repl substr($0, RSTART + RLENGTH)
+ }
+
+ while (match($0, /'\''__SECRET_[A-Za-z_][A-Za-z0-9_]*__'\''/)) {
+ token = substr($0, RSTART + 1, RLENGTH - 2)
+ key = substr(token, 10, length(token) - 11)
+
+ if (key in secrets) {
+ repl = yaml_quote(secrets[key])
+ } else {
+ repl = yaml_quote("")
+ if (!(token in missing)) {
+ missing[token] = 1
+ missing_order[++missing_count] = token
+ }
+ }
+
+ $0 = substr($0, 1, RSTART - 1) repl substr($0, RSTART + RLENGTH)
+ }
+
+ if (match($0, /^[[:space:]]*-[[:space:]]*__SECRET_API_KEY_STRATEGIES__[[:space:]]*$/)) {
+ indent = substr($0, 1, index($0, "-") - 1)
+ emit_api_key_strategies(indent)
+ next
+ }
+
+ if (match($0, /^[[:space:]]*__SECRET_API_KEY_STRATEGIES__[[:space:]]*$/)) {
+ indent = $0
+ sub(/__SECRET_API_KEY_STRATEGIES__.*/, "", indent)
+ emit_api_key_strategies(indent)
+ next
+ }
+
+ out = ""
+ rest = $0
+ while (match(rest, /__SECRET_[A-Za-z_][A-Za-z0-9_]*__/)) {
+ token = substr(rest, RSTART, RLENGTH)
+ key = substr(token, 10, length(token) - 11)
+
+ if (key in secrets) {
+ repl = secrets[key]
+ } else {
+ repl = token
+ if (!(token in missing)) {
+ missing[token] = 1
+ missing_order[++missing_count] = token
+ }
+ }
+
+ out = out substr(rest, 1, RSTART - 1) repl
+ rest = substr(rest, RSTART + RLENGTH)
+ }
+ print out rest
+ }
+ END {
+ if (missing_count > 0) {
+ print "ERROR: Unresolved ERPC secret placeholders remain in rendered config:" > "/dev/stderr"
+ for (i = 1; i <= missing_count; i++) {
+ print missing_order[i] > "/dev/stderr"
+ }
+ exit 1
+ }
+ }
+ ' /vault/secrets/erpc-config > "$tmp_rendered_config"
+ mv "$tmp_rendered_config" "$rendered_config"
+
+ echo "ERPC configuration rendered to $rendered_config"
+ {{- if $useDedicatedValidator }}
+ - name: config-validator
+ {{- if .Values.securityHardening.enabled }}
+ securityContext:
+ {{- toYaml .Values.securityHardening.containerSecurityContext | nindent 12 }}
+ {{- end }}
+ image: "{{ $validationImage.repository }}:{{ $validationImage.tag }}"
+ imagePullPolicy: {{ $validationImage.pullPolicy | default .Values.vault.jobImage.pullPolicy }}
+ {{- if .Values.securityHardening.enabled }}
+ env:
+ - name: HOME
+ value: /tmp
+ {{- end }}
+ volumeMounts:
+ - name: rendered-config
+ mountPath: /work
+ {{- if .Values.securityHardening.enabled }}
+ - name: tmp
+ mountPath: /tmp
+ {{- end }}
+ command:
+ {{- toYaml ($validationImage.command | default (list "erpc")) | nindent 12 }}
+ args:
+ - validate
+ - --config
+ - /work/erpc.yaml
+ resources:
+ {{- toYaml ($validationImage.resources | default .Values.vault.jobResources) | nindent 12 }}
+ {{- end }}
+ containers:
+ - name: config-creator
+ {{- if .Values.securityHardening.enabled }}
+ securityContext:
+ {{- toYaml .Values.securityHardening.containerSecurityContext | nindent 12 }}
+ {{- end }}
+ image: "{{ .Values.vault.jobImage.repository }}:{{ .Values.vault.jobImage.tag }}"
+ imagePullPolicy: {{ .Values.vault.jobImage.pullPolicy }}
+ resources:
+ {{- toYaml .Values.vault.jobResources | nindent 12 }}
+ {{- if .Values.securityHardening.enabled }}
+ env:
+ - name: HOME
+ value: /tmp
+ {{- end }}
+ volumeMounts:
+ - name: rendered-config
+ mountPath: /work
+ {{- if .Values.securityHardening.enabled }}
+ - name: tmp
+ mountPath: /tmp
+ {{- end }}
+ command:
+ - sh
+ - -c
+ - |
+ set -eu
+ if (set -o pipefail) 2>/dev/null; then
+ set -o pipefail
+ fi
+
+ rendered_config=/work/erpc.yaml
+ echo "Creating Kubernetes Secret from rendered vault config..."
+
+ if [ ! -s "$rendered_config" ]; then
+ echo "ERROR: Rendered ERPC config file not found or empty: $rendered_config"
+ exit 1
+ fi
+
+ {{- if not $useDedicatedValidator }}
+ echo "Validating ERPC configuration..."
+
+ # Validate ERPC configuration using erpc binary
+ if ! erpc validate --config "$rendered_config"; then
+ echo "ERROR: Invalid ERPC configuration"
+ exit 1
+ fi
+
+ echo "ERPC configuration validation passed. Creating Kubernetes Secret..."
+ {{- else }}
+ echo "ERPC configuration validation passed with dedicated runtime image. Creating Kubernetes Secret..."
+ {{- end }}
+
+ # Create secret from rendered vault config file
+ kubectl create secret generic {{ include "erpc.fullname" . }}-vault-config \
+ --namespace={{ .Release.Namespace }} \
+ --from-file=erpc.yaml="$rendered_config" \
+ --dry-run=client -o yaml | kubectl apply -f -
+
+ echo "Secret {{ include "erpc.fullname" . }}-vault-config created successfully"
+
+ # Verify secret was created
+ kubectl get secret {{ include "erpc.fullname" . }}-vault-config -n {{ .Release.Namespace }}
diff --git a/helm/charts/erpc/templates/job-vault-db.yaml b/helm/charts/erpc/templates/job-vault-db.yaml
new file mode 100644
index 000000000..83eb51a47
--- /dev/null
+++ b/helm/charts/erpc/templates/job-vault-db.yaml
@@ -0,0 +1,87 @@
+{{- if ne (toString .Values.postgresql.enabled) "false" }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ include "erpc.fullname" . }}-db-secret-creator
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+ annotations:
+ "helm.sh/hook": pre-install,pre-upgrade
+ "helm.sh/hook-weight": "0"
+ "helm.sh/hook-delete-policy": hook-succeeded
+ argocd.argoproj.io/sync-wave: "-3"
+ argocd.argoproj.io/sync-options: Force=true,Replace=true
+spec:
+ template:
+ metadata:
+ labels:
+ {{- include "erpc.selectorLabels" . | nindent 8 }}
+ job: db-secret-creator
+ annotations:
+ vault.hashicorp.com/agent-inject: "true"
+ vault.hashicorp.com/role: {{ include "erpc.vaultCreatorServiceAccountName" . | quote }}
+ vault.hashicorp.com/agent-inject-secret-secrets: {{ .Values.postgresql.vault.secretsPath | default "secret/data/erpc/db" | quote }}
+ vault.hashicorp.com/agent-inject-template-secrets: |
+ {{`{{- with secret "`}}{{ .Values.postgresql.vault.secretsPath | default "secret/data/erpc/db" }}{{`" -}}
+ {{- .Data.data.password -}}
+ {{- end }}`}}
+ vault.hashicorp.com/agent-pre-populate-only: "true"
+ spec:
+ serviceAccountName: {{ include "erpc.vaultCreatorServiceAccountName" . }}
+ automountServiceAccountToken: true
+ {{- if .Values.securityHardening.enabled }}
+ securityContext:
+ {{- toYaml .Values.securityHardening.podSecurityContext | nindent 8 }}
+ {{- end }}
+ restartPolicy: OnFailure
+ {{- if .Values.securityHardening.enabled }}
+ volumes:
+ - name: tmp
+ emptyDir: {}
+ {{- end }}
+ containers:
+ - name: db-secret-creator
+ {{- if .Values.securityHardening.enabled }}
+ securityContext:
+ {{- toYaml .Values.securityHardening.containerSecurityContext | nindent 12 }}
+ {{- end }}
+ image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/alpine/k8s:1.31.0
+ {{- if .Values.securityHardening.enabled }}
+ env:
+ - name: HOME
+ value: /tmp
+ volumeMounts:
+ - name: tmp
+ mountPath: /tmp
+ {{- end }}
+ command:
+ - sh
+ - -c
+ - |
+ echo "Creating ERPC database Kubernetes Secret from Vault..."
+
+ timeout=60
+ count=0
+ while [ ! -f /vault/secrets/secrets ] && [ $count -lt $timeout ]; do
+ echo "Waiting for vault secrets file... ($count/$timeout)"
+ sleep 2
+ count=$((count + 2))
+ done
+
+ if [ ! -f /vault/secrets/secrets ]; then
+ echo "ERROR: Vault secrets file not found after $timeout seconds"
+ exit 1
+ fi
+
+ echo "Vault secrets file found. Creating Kubernetes Secret..."
+
+ kubectl create secret generic erpc-db-secret \
+ --namespace={{ .Release.Namespace }} \
+ --from-literal=username=postgres \
+ --from-file=password=/vault/secrets/secrets \
+ --dry-run=client -o yaml | kubectl apply -f -
+
+ echo "Secret erpc-db-secret created successfully"
+
+ kubectl get secret erpc-db-secret -n {{ .Release.Namespace }}
+{{- end }}
diff --git a/helm/charts/erpc/templates/monitoring-configmap.yaml b/helm/charts/erpc/templates/monitoring-configmap.yaml
new file mode 100644
index 000000000..45356facc
--- /dev/null
+++ b/helm/charts/erpc/templates/monitoring-configmap.yaml
@@ -0,0 +1,59 @@
+{{- if ne (toString .Values.postgresql.enabled) "false" }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "erpc.fullname" . }}-cnpg-recovery-health-monitoring
+ labels:
+ cnpg.io/reload: ""
+ {{- include "erpc.labels" . | nindent 4 }}
+data:
+ custom-queries: |
+ # PostgreSQL Recovery Health Metrics
+ # Detects conditions that prevent hot standby or cause recovery issues
+
+ pg_recovery_health:
+ query: |
+ SELECT
+ CASE WHEN pg_is_in_recovery() THEN 1 ELSE 0 END AS is_replica,
+ CASE WHEN pg_is_in_recovery() AND NOT pg_is_wal_replay_paused() THEN 1 ELSE 0 END AS recovery_active,
+ CASE WHEN pg_is_in_recovery() AND pg_is_wal_replay_paused() THEN 1 ELSE 0 END AS recovery_paused,
+ COALESCE(EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::bigint, 0) AS replication_lag_seconds
+ metrics:
+ - is_replica:
+ usage: "GAUGE"
+ description: "1 if this instance is a replica, 0 if primary"
+ - recovery_active:
+ usage: "GAUGE"
+ description: "1 if recovery is actively replaying WAL, 0 otherwise"
+ - recovery_paused:
+ usage: "GAUGE"
+ description: "1 if recovery is paused (critical - server will shut down if unpaused)"
+ - replication_lag_seconds:
+ usage: "GAUGE"
+ description: "Seconds since last replayed transaction"
+
+ pg_hot_standby_params:
+ query: |
+ SELECT
+ current_setting('max_connections')::int AS max_connections,
+ current_setting('max_worker_processes')::int AS max_worker_processes,
+ current_setting('max_wal_senders')::int AS max_wal_senders,
+ current_setting('max_prepared_transactions')::int AS max_prepared_transactions,
+ current_setting('max_locks_per_transaction')::int AS max_locks_per_transaction
+ metrics:
+ - max_connections:
+ usage: "GAUGE"
+ description: "Current max_connections setting"
+ - max_worker_processes:
+ usage: "GAUGE"
+ description: "Current max_worker_processes setting"
+ - max_wal_senders:
+ usage: "GAUGE"
+ description: "Current max_wal_senders setting"
+ - max_prepared_transactions:
+ usage: "GAUGE"
+ description: "Current max_prepared_transactions setting"
+ - max_locks_per_transaction:
+ usage: "GAUGE"
+ description: "Current max_locks_per_transaction setting"
+{{- end }}
diff --git a/helm/charts/erpc/templates/pdb.yaml b/helm/charts/erpc/templates/pdb.yaml
new file mode 100644
index 000000000..3e9a5fe2c
--- /dev/null
+++ b/helm/charts/erpc/templates/pdb.yaml
@@ -0,0 +1,12 @@
+{{- $replicas := ternary .Values.autoscaling.minReplicas .Values.replicaCount .Values.autoscaling.enabled -}}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+ name: {{ include "erpc.fullname" . }}-pdb
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+spec:
+ minAvailable: {{ sub $replicas 1 }}
+ selector:
+ matchLabels:
+ {{- include "erpc.selectorLabels" . | nindent 6 }}
diff --git a/helm/charts/erpc/templates/pg-erpc.yaml b/helm/charts/erpc/templates/pg-erpc.yaml
new file mode 100644
index 000000000..e86f43667
--- /dev/null
+++ b/helm/charts/erpc/templates/pg-erpc.yaml
@@ -0,0 +1,186 @@
+{{- if ne (toString .Values.postgresql.enabled) "false" }}
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+ name: erpc-db
+ annotations:
+ argocd.argoproj.io/sync-wave: "-2"
+spec:
+ instances: 2
+ primaryUpdateStrategy: unsupervised
+ imageCatalogRef:
+ apiGroup: postgresql.cnpg.io
+ kind: ImageCatalog
+ name: erpc-postgresql-native-18
+ major: 18
+
+ serviceAccountTemplate:
+ metadata:
+ annotations:
+ eks.amazonaws.com/role-arn: {{ .Values.postgresql.s3SARoleArn }}
+ postgresql:
+ shared_preload_libraries:
+ - pg_stat_statements
+ parameters:
+ max_connections: "5000"
+
+ idle_in_transaction_session_timeout: "30000"
+
+ max_locks_per_transaction: "512"
+ max_pred_locks_per_transaction: "512"
+
+ shared_buffers: {{ .Values.postgresql.sharedBuffers | default "1GB" | quote }}
+ effective_cache_size: {{ .Values.postgresql.effectiveCacheSize | default "3GB" | quote }}
+ maintenance_work_mem: {{ .Values.postgresql.maintenanceWorkMem | default "256MB" | quote }}
+ work_mem: {{ .Values.postgresql.workMem | default "8MB" | quote }}
+
+ wal_buffers: "16MB"
+ min_wal_size: "1GB"
+ max_wal_size: "8GB"
+ wal_keep_size: "2GB"
+ checkpoint_completion_target: "0.9"
+ checkpoint_timeout: "15min"
+ wal_level: "replica"
+ synchronous_commit: "off"
+
+ default_statistics_target: "200"
+ random_page_cost: "1.0"
+ effective_io_concurrency: "200"
+ huge_pages: "off"
+
+ max_worker_processes: {{ .Values.postgresql.maxWorkerProcesses | default "2" | quote }}
+ max_parallel_workers: {{ .Values.postgresql.maxParallelWorkers | default "2" | quote }}
+ max_parallel_workers_per_gather: {{ .Values.postgresql.maxParallelWorkersPerGather | default "1" | quote }}
+ max_parallel_maintenance_workers: {{ .Values.postgresql.maxParallelMaintenanceWorkers | default "1" | quote }}
+
+ autovacuum_max_workers: {{ .Values.postgresql.autovacuumMaxWorkers | default "2" | quote }}
+ autovacuum_naptime: "10s"
+ autovacuum_vacuum_scale_factor: "0.01"
+ autovacuum_analyze_scale_factor: "0.005"
+ autovacuum_vacuum_cost_delay: "2ms"
+ autovacuum_vacuum_cost_limit: "2000"
+
+ track_io_timing: "on"
+ jit: "off"
+
+ bgwriter_delay: "200ms"
+ bgwriter_lru_maxpages: "1000"
+ bgwriter_lru_multiplier: "10.0"
+
+ track_activity_query_size: "8192"
+ pg_stat_statements.max: "5000"
+ pg_stat_statements.track: "all"
+
+ # Must exceed worst-case pg_backup_start() duration (checkpoint + WAL flush)
+ # on a write-heavy DB; 10s was killing every barman base backup.
+ statement_timeout: "600000"
+ lock_timeout: "5000"
+
+ vacuum_cost_delay: "2ms"
+
+ # Write-heavy workload optimizations
+ full_page_writes: "on"
+ wal_compression: "on"
+ wal_writer_delay: "10ms"
+ commit_delay: "10"
+ commit_siblings: "10"
+
+ # Reduce checkpoint frequency impact
+ checkpoint_warning: "300s"
+
+ # Connection pooling optimization
+ tcp_keepalives_idle: "60"
+ tcp_keepalives_interval: "10"
+ tcp_keepalives_count: "6"
+ pg_hba:
+ - host all all 0.0.0.0/0 md5
+
+ bootstrap:
+ initdb:
+ database: erpc
+ owner: postgres
+ secret:
+ name: erpc-db-secret
+ postInitApplicationSQL:
+ - CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
+
+ storage:
+ storageClass: {{ .Values.postgresql.storageClass | default "gp3" }}
+ size: {{ .Values.postgresql.storage | default "100Gi" }}
+ {{- with .Values.postgresql.volumeAttributesClassName }}
+ pvcTemplate:
+ volumeAttributesClassName: {{ . | quote }}
+ {{- end }}
+
+ resources:
+ {{- toYaml .Values.postgresql.resources | nindent 4 }}
+
+ {{- $aff := .Values.postgresql.affinity | default dict -}}
+ {{- if or $aff.nodeSelector $aff.tolerations $aff.additionalPodAntiAffinity }}
+ affinity:
+ {{- with $aff.nodeSelector }}
+ nodeSelector:
+ {{- toYaml . | nindent 6 }}
+ {{- end }}
+ {{- with $aff.tolerations }}
+ tolerations:
+ {{- toYaml . | nindent 6 }}
+ {{- end }}
+ {{- with $aff.additionalPodAntiAffinity }}
+ additionalPodAntiAffinity:
+ {{- toYaml . | nindent 6 }}
+ {{- end }}
+ {{- else }}
+ affinity: {}
+ {{- end }}
+
+ enableSuperuserAccess: true
+ superuserSecret:
+ name: erpc-db-secret
+
+ monitoring:
+ customQueriesConfigMap:
+ - key: queries
+ name: cnpg-default-monitoring
+ - key: custom-queries
+ name: {{ include "erpc.fullname" . }}-cnpg-recovery-health-monitoring
+
+ {{- $snapshotClass := .Values.postgresql.backup.volumeSnapshotClassName | default "" }}
+ {{- $backupMethod := .Values.postgresql.backup.method | default (ternary "volumeSnapshot" "barmanObjectStore" (ne $snapshotClass "")) }}
+ {{- if eq $backupMethod "volumeSnapshot" }}
+ {{- if not $snapshotClass }}{{ fail "postgresql.backup.volumeSnapshotClassName is required when postgresql.backup.method=volumeSnapshot" }}{{ end }}
+ backup:
+ retentionPolicy: {{ .Values.postgresql.backup.retentionPolicy | default "7d" | quote }}
+ {{- with .Values.postgresql.backup.destinationPath }}
+ barmanObjectStore:
+ s3Credentials:
+ inheritFromIAMRole: true
+ destinationPath: {{ . }}
+ endpointURL: "https://s3.eu-west-3.amazonaws.com"
+ wal:
+ compression: gzip
+ maxParallel: 4
+ {{- end }}
+ target: prefer-standby
+ volumeSnapshot:
+ className: {{ $snapshotClass }}
+ online: true
+ onlineConfiguration:
+ immediateCheckpoint: false
+ waitForArchive: true
+ snapshotOwnerReference: backup
+ {{- else if eq $backupMethod "barmanObjectStore" }}
+ backup:
+ retentionPolicy: {{ .Values.postgresql.backup.retentionPolicy | default "7d" | quote }}
+ barmanObjectStore:
+ s3Credentials:
+ inheritFromIAMRole: true
+ destinationPath: {{ .Values.postgresql.backup.destinationPath }}
+ endpointURL: "https://s3.eu-west-3.amazonaws.com"
+ wal:
+ compression: gzip
+ maxParallel: 4
+ {{- else }}
+ {{- fail (printf "postgresql.backup.method must be \"barmanObjectStore\" or \"volumeSnapshot\", got %q" $backupMethod) }}
+ {{- end }}
+{{- end }}
diff --git a/helm/charts/erpc/templates/podmonitor-erpc-db.yaml b/helm/charts/erpc/templates/podmonitor-erpc-db.yaml
new file mode 100644
index 000000000..fc0860b8a
--- /dev/null
+++ b/helm/charts/erpc/templates/podmonitor-erpc-db.yaml
@@ -0,0 +1,17 @@
+{{- if ne (toString .Values.postgresql.enabled) "false" }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+ name: erpc-db-monitor
+ labels:
+ app: erpc-db-monitor
+ cnpg-cluster: erpc-db
+spec:
+ selector:
+ matchLabels:
+ cnpg.io/cluster: erpc-db
+ podMetricsEndpoints:
+ - port: metrics
+ interval: 10s
+ scrapeTimeout: 8s
+{{- end }}
diff --git a/helm/charts/erpc/templates/podmonitor.yaml b/helm/charts/erpc/templates/podmonitor.yaml
new file mode 100644
index 000000000..627f8372d
--- /dev/null
+++ b/helm/charts/erpc/templates/podmonitor.yaml
@@ -0,0 +1,61 @@
+{{- if .Values.monitoring.podMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+ name: {{ include "erpc.fullname" . }}
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+ {{- with .Values.monitoring.podMonitor.additionalLabels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ selector:
+ matchLabels:
+ {{- include "erpc.selectorLabels" . | nindent 6 }}
+ podMetricsEndpoints:
+ - port: metrics
+ {{- with .Values.monitoring.podMonitor.interval }}
+ interval: {{ . }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.scrapeTimeout }}
+ scrapeTimeout: {{ . }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.path }}
+ path: {{ . }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.scheme }}
+ scheme: {{ . }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.honorLabels }}
+ honorLabels: {{ . }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.metricRelabelings }}
+ metricRelabelings:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.relabelings }}
+ relabelings:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.namespaceSelector }}
+ namespaceSelector:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.jobLabel }}
+ jobLabel: {{ . }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.podTargetLabels }}
+ podTargetLabels:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.sampleLimit }}
+ sampleLimit: {{ . }}
+ {{- end }}
+ {{- with .Values.monitoring.podMonitor.targetLimit }}
+ targetLimit: {{ . }}
+ {{- end }}
+{{- end }}
\ No newline at end of file
diff --git a/helm/charts/erpc/templates/rbac-vault.yaml b/helm/charts/erpc/templates/rbac-vault.yaml
new file mode 100644
index 000000000..55391c698
--- /dev/null
+++ b/helm/charts/erpc/templates/rbac-vault.yaml
@@ -0,0 +1,35 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ include "erpc.fullname" . }}-vault-secret
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+ annotations:
+ argocd.argoproj.io/sync-wave: "-4"
+rules:
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["create"]
+ - apiGroups: [""]
+ resources: ["secrets"]
+ resourceNames:
+ {{- include "erpc.vaultSecretResourceNames" . | nindent 6 }}
+ verbs: ["get", "patch", "update"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ include "erpc.fullname" . }}-vault-secret
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+ annotations:
+ argocd.argoproj.io/sync-wave: "-4"
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ include "erpc.fullname" . }}-vault-secret
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "erpc.vaultCreatorServiceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
diff --git a/helm/charts/erpc/templates/scheduled-backup.yaml b/helm/charts/erpc/templates/scheduled-backup.yaml
new file mode 100644
index 000000000..d50daf0fc
--- /dev/null
+++ b/helm/charts/erpc/templates/scheduled-backup.yaml
@@ -0,0 +1,20 @@
+{{- if ne (toString .Values.postgresql.enabled) "false" }}
+{{- $snapshotClass := .Values.postgresql.backup.volumeSnapshotClassName | default "" }}
+{{- $backupMethod := .Values.postgresql.backup.method | default (ternary "volumeSnapshot" "barmanObjectStore" (ne $snapshotClass "")) }}
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+ name: erpc-db-scheduled-backup
+spec:
+ {{- if eq $backupMethod "volumeSnapshot" }}
+ method: volumeSnapshot
+ {{- else if eq $backupMethod "barmanObjectStore" }}
+ method: barmanObjectStore
+ {{- else }}
+ {{- fail (printf "postgresql.backup.method must be \"barmanObjectStore\" or \"volumeSnapshot\", got %q" $backupMethod) }}
+ {{- end }}
+ schedule: "0 0 0 * * *"
+ backupOwnerReference: self
+ cluster:
+ name: erpc-db
+{{- end }}
diff --git a/helm/charts/erpc/templates/service-headless.yaml b/helm/charts/erpc/templates/service-headless.yaml
new file mode 100644
index 000000000..fbbef67ab
--- /dev/null
+++ b/helm/charts/erpc/templates/service-headless.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.headlessService.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "erpc.fullname" . }}-headless
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+spec:
+ type: ClusterIP
+ clusterIP: None
+ publishNotReadyAddresses: false
+ ports:
+ - port: {{ .Values.service.port }}
+ targetPort: http
+ protocol: TCP
+ name: http
+ - port: {{ .Values.service.metricsPort }}
+ targetPort: metrics
+ protocol: TCP
+ name: metrics
+ selector:
+ {{- include "erpc.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/helm/charts/erpc/templates/service-metrics.yaml b/helm/charts/erpc/templates/service-metrics.yaml
new file mode 100644
index 000000000..0f99693ea
--- /dev/null
+++ b/helm/charts/erpc/templates/service-metrics.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "erpc.fullname" . }}-metrics
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+ app.kubernetes.io/component: metrics
+ {{- with .Values.metricsService.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ type: {{ .Values.metricsService.type }}
+ ports:
+ - port: {{ .Values.service.metricsPort }}
+ targetPort: metrics
+ protocol: TCP
+ name: metrics
+ selector:
+ {{- include "erpc.selectorLabels" . | nindent 4 }}
diff --git a/helm/charts/erpc/templates/service.yaml b/helm/charts/erpc/templates/service.yaml
new file mode 100644
index 000000000..466c77f1c
--- /dev/null
+++ b/helm/charts/erpc/templates/service.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "erpc.fullname" . }}
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+ {{- with .Values.service.annotations }}
+ annotations:
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ type: {{ .Values.service.type }}
+ ports:
+ - port: {{ .Values.service.port }}
+ targetPort: http
+ protocol: TCP
+ name: http
+ selector:
+ {{- include "erpc.selectorLabels" . | nindent 4 }}
\ No newline at end of file
diff --git a/helm/charts/erpc/templates/serviceaccount.yaml b/helm/charts/erpc/templates/serviceaccount.yaml
new file mode 100644
index 000000000..0b5cf03c9
--- /dev/null
+++ b/helm/charts/erpc/templates/serviceaccount.yaml
@@ -0,0 +1,44 @@
+{{- $serviceAccountCreate := ne (toString .Values.serviceAccount.create) "false" -}}
+{{- $vaultCreator := .Values.serviceAccount.vaultCreator | default dict -}}
+{{- $hasCreateOverride := hasKey $vaultCreator "create" -}}
+{{- $vaultCreatorName := $vaultCreator.name | default "" -}}
+{{- $hasExplicitCreatorName := ne (toString $vaultCreatorName) "" -}}
+{{- $createVaultCreator := and $serviceAccountCreate (not $hasExplicitCreatorName) -}}
+{{- if $hasCreateOverride -}}
+{{- $createVaultCreator = ne (toString $vaultCreator.create) "false" -}}
+{{- end -}}
+{{- $runtimeName := .Values.serviceAccount.runtimeName | default "" -}}
+{{- $createRuntime := or $serviceAccountCreate (ne (toString $runtimeName) "") -}}
+{{- $runtimeServiceAccountName := include "erpc.serviceAccountName" . -}}
+{{- $vaultCreatorServiceAccountName := include "erpc.vaultCreatorServiceAccountName" . -}}
+{{- if $createRuntime }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ $runtimeServiceAccountName }}
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+ annotations:
+ argocd.argoproj.io/sync-wave: "-4"
+ {{- with .Values.serviceAccount.annotations }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+automountServiceAccountToken: false
+{{- end }}
+{{- if and $createVaultCreator (or (not $createRuntime) (ne $vaultCreatorServiceAccountName $runtimeServiceAccountName)) }}
+{{- if $createRuntime }}
+---
+{{- end }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ $vaultCreatorServiceAccountName }}
+ labels:
+ {{- include "erpc.labels" . | nindent 4 }}
+ annotations:
+ argocd.argoproj.io/sync-wave: "-4"
+ {{- with .Values.serviceAccount.annotations }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+automountServiceAccountToken: false
+{{- end }}
diff --git a/helm/charts/erpc/test-local.sh b/helm/charts/erpc/test-local.sh
new file mode 100755
index 000000000..e9d3b1ecf
--- /dev/null
+++ b/helm/charts/erpc/test-local.sh
@@ -0,0 +1,275 @@
+#!/bin/bash
+# eRPC Local Docker Compose Test Script (DynamoDB Configuration)
+# Based on: https://docs.erpc.cloud/
+
+set -e
+
+cd "$(dirname "$0")"
+
+# Source the .env file to get environment variables
+if [ -f .env ]; then
+ source .env
+fi
+
+echo "🚀 Testing eRPC with Docker Compose (DynamoDB configuration)..."
+
+# Check if docker-compose is available
+if ! command -v docker-compose &> /dev/null || ! command -v docker &> /dev/null; then
+ echo "❌ Docker and docker-compose are required but not installed"
+ exit 1
+fi
+
+# Check if .env file exists
+if [ ! -f .env ]; then
+ echo "⚠️ No .env file found. Creating from example..."
+ cp .env.example .env
+ echo "📝 Please edit .env file with your actual API keys:"
+ echo " - ALCHEMY_API_KEY"
+ echo " - DRPC_API_KEY"
+ echo " - TENDERLY_API_KEY"
+ echo " - TENDERLY_PROJECT"
+ echo " - LABELS_SERVICE_API_KEY (for authentication)"
+ echo " - ERPC_API_KEY (for Goldsky edge access)"
+ echo ""
+ echo "⚠️ Note: This configuration uses DynamoDB for caching and shared state."
+ echo " You'll need AWS credentials or IAM role configured for DynamoDB access."
+ echo ""
+ echo "Then run this script again."
+ exit 1
+fi
+
+# Start services (only eRPC, no Redis needed)
+echo "🐳 Starting eRPC service..."
+docker-compose up -d erpc
+
+# Wait for services to be ready
+echo "⏳ Waiting for services to be ready..."
+for i in {1..30}; do
+ # Check if eRPC container is running
+ if docker-compose ps | grep "erpc-erpc" | grep -q "Up"; then
+ # Test if port 4000 is accessible
+ if nc -z localhost 4000 2>/dev/null; then
+ # Test if health endpoint responds with OK
+ health_response=$(curl -s http://localhost:4000/healthcheck 2>/dev/null || echo "failed")
+ if echo "$health_response" | grep -q '"status":"OK"'; then
+ echo "✅ eRPC is ready and responding!"
+ echo " Providers: $(echo "$health_response" | jq -r '.details.totalProviders // "unknown"' 2>/dev/null)"
+ break
+ else
+ echo " eRPC port open but health check not ready... ($i/30)"
+ echo " Response: $(echo "$health_response" | head -c 100)..."
+ fi
+ else
+ echo " eRPC container up but port 4000 not ready... ($i/30)"
+ fi
+ else
+ echo " eRPC container not up yet... ($i/30)"
+ fi
+
+ if [ $i -eq 30 ]; then
+ echo "❌ eRPC failed to start within timeout"
+ echo "🔍 Container status:"
+ docker-compose ps
+ echo "🔍 eRPC logs:"
+ docker-compose logs erpc
+ exit 1
+ fi
+ sleep 2
+done
+
+# Cleanup function
+cleanup() {
+ echo "🧹 Cleaning up..."
+ docker-compose down
+}
+trap cleanup EXIT
+
+# Test 1: Health check
+echo "🏥 Testing health endpoint..."
+health_response=$(curl -s http://localhost:4000/healthcheck 2>/dev/null || echo "failed")
+
+if echo "$health_response" | grep -q '"status":"OK"'; then
+ echo "✅ Health check passed"
+ echo " Status: $(echo "$health_response" | jq -r '.message' 2>/dev/null || echo "OK")"
+else
+ echo "❌ Health check failed"
+ echo " Response: $health_response"
+ exit 1
+fi
+
+# Test 2: Ethereum RPC call
+echo "🌐 Testing Ethereum RPC call..."
+response=$(curl -s --location "http://localhost:4000/cache/evm/1?secret=${LABELS_SERVICE_API_KEY}" \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "method": "eth_blockNumber",
+ "params": [],
+ "id": 1,
+ "jsonrpc": "2.0"
+ }')
+
+if echo "$response" | grep -q '"result"'; then
+ block_number=$(echo "$response" | jq -r '.result' 2>/dev/null || echo "unknown")
+ echo "✅ Ethereum RPC call successful"
+ echo " Latest block: $block_number"
+else
+ echo "❌ Ethereum RPC call failed"
+ echo " Response: $response"
+ exit 1
+fi
+
+# Test 3: Arbitrum RPC call (from eRPC quickstart docs)
+echo "🔗 Testing Arbitrum RPC call..."
+response=$(curl -s --location "http://localhost:4000/cache/evm/42161?secret=${LABELS_SERVICE_API_KEY}" \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "method": "eth_getBlockByNumber",
+ "params": ["latest", false],
+ "id": 9199,
+ "jsonrpc": "2.0"
+ }')
+
+if echo "$response" | grep -q '"result"'; then
+ echo "✅ Arbitrum RPC call successful"
+ block_hash=$(echo "$response" | jq -r '.result.hash // "unknown"' 2>/dev/null || echo "unknown")
+ echo " Block hash: $block_hash"
+else
+ echo "❌ Arbitrum RPC call failed"
+ echo " Response: $response"
+ exit 1
+fi
+
+# Test 4: Base network RPC call
+echo "🔵 Testing Base network RPC call..."
+response=$(curl -s --location "http://localhost:4000/cache/evm/8453?secret=${LABELS_SERVICE_API_KEY}" \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "method": "eth_chainId",
+ "params": [],
+ "id": 1,
+ "jsonrpc": "2.0"
+ }')
+
+if echo "$response" | grep -q '"result"'; then
+ chain_id=$(echo "$response" | jq -r '.result' 2>/dev/null || echo "unknown")
+ expected_chain_id="0x2105" # 8453 in hex
+ echo "✅ Base network RPC call successful"
+ echo " Chain ID: $chain_id (expected: $expected_chain_id)"
+else
+ echo "❌ Base network RPC call failed"
+ echo " Response: $response"
+ exit 1
+fi
+
+# Test 5: Authentication test (if LABELS_SERVICE_API_KEY is configured)
+echo "🔐 Testing authentication..."
+
+if [ -n "${LABELS_SERVICE_API_KEY:-}" ] && [ "$LABELS_SERVICE_API_KEY" != "your-secure-auth-secret-here" ]; then
+ # Test without auth secret (should fail if auth is enabled)
+ auth_response_no_secret=$(curl -s --location 'http://localhost:4000/cache/evm/1' \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "method": "eth_blockNumber",
+ "params": [],
+ "id": 1,
+ "jsonrpc": "2.0"
+ }')
+
+ if echo "$auth_response_no_secret" | grep -q "unauthorized\|auth"; then
+ echo "✅ Authentication is properly enforced (request without secret rejected)"
+
+ # Test with proper auth secret in URL
+ auth_response_with_secret=$(curl -s --location "http://localhost:4000/cache/evm/1?secret=$LABELS_SERVICE_API_KEY" \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "method": "eth_blockNumber",
+ "params": [],
+ "id": 1,
+ "jsonrpc": "2.0"
+ }')
+
+ if echo "$auth_response_with_secret" | grep -q '"result"'; then
+ echo "✅ Authentication with proper secret parameter works"
+ else
+ echo "⚠️ Authentication secret parameter didn't work as expected"
+ # Redact the API key in case the response (or a reflected error)
+ # echoes back the request URL containing the secret.
+ echo " Response: ${auth_response_with_secret//$LABELS_SERVICE_API_KEY/***REDACTED***}"
+ fi
+ else
+ echo "⚠️ Authentication might not be properly configured (no auth rejection detected)"
+ echo " Response: $auth_response_no_secret"
+ fi
+else
+ echo "ℹ️ Authentication not configured or using default value"
+fi
+
+# Test 6: Arbitrum eth_getLogs test (specific range and topics)
+echo "📋 Testing Arbitrum eth_getLogs with specific range and topics..."
+response=$(curl -s --location "http://localhost:4000/cache/evm/42161?secret=${LABELS_SERVICE_API_KEY}" \
+ --header 'Content-Type: application/json' \
+ --data '{
+ "method": "eth_getLogs",
+ "params": [{
+ "fromBlock": "0x1254048f",
+ "toBlock": "0x12558b2f",
+ "topics": [["0xe0c2db6b54586be6d7d49943139fccf0dd315ba63e55364a76c73cd8fdba724d","0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0"]],
+ "address": ["0x35a04d797164d54def24d6fc722c8f82f8ef0d45","0x1350994173f1cc83f8fc45a5af80a0acfc1b613c"]
+ }],
+ "id": 1,
+ "jsonrpc": "2.0"
+ }')
+
+if echo "$response" | grep -q '"result"'; then
+ log_count=$(echo "$response" | jq -r '.result | length' 2>/dev/null || echo "unknown")
+ echo "✅ Arbitrum eth_getLogs test successful"
+ echo " Logs found: $log_count"
+ echo " Block range: 0x1254048f (307,496,079) to 0x12558b2f (307,596,079)"
+ echo " Range size: 100,000 blocks"
+elif echo "$response" | grep -q '"error"'; then
+ error_message=$(echo "$response" | jq -r '.error.message // .error' 2>/dev/null || echo "unknown")
+ echo "⚠️ eth_getLogs returned error (expected for large ranges):"
+ echo " Error: $error_message"
+ # This is actually expected behavior for large block ranges - eRPC should auto-split
+else
+ echo "❌ Arbitrum eth_getLogs test failed"
+ echo " Response: $response"
+ exit 1
+fi
+
+# Test 7: DynamoDB cache verification
+echo "💾 DynamoDB cache configuration:"
+echo "✅ DynamoDB caching enabled"
+echo " Note: DynamoDB access requires AWS credentials or IAM role"
+echo " Table pattern: {environment}-erpc-cache"
+
+echo ""
+echo "🎉 All eRPC local tests passed!"
+echo ""
+echo "📋 Service URLs:"
+echo " 🌐 RPC Proxy: http://localhost:4000/cache/evm/{chainId}"
+echo " 📊 Metrics: http://localhost:4001/metrics"
+echo " 💾 Cache: DynamoDB (configured via environment)"
+echo ""
+echo "🔗 Supported networks (all EVM chains via networkDefaults):"
+echo " - Ethereum: http://localhost:4000/cache/evm/1"
+echo " - Base: http://localhost:4000/cache/evm/8453"
+echo " - Polygon: http://localhost:4000/cache/evm/137"
+echo " - Optimism: http://localhost:4000/cache/evm/10"
+echo " - Arbitrum: http://localhost:4000/cache/evm/42161"
+echo " - BSC: http://localhost:4000/cache/evm/56"
+echo " - + Any other EVM-compatible network"
+echo ""
+echo "🐛 Debug calls automatically route to Tenderly:"
+echo " curl --location 'http://localhost:4000/cache/evm/1' \\"
+echo " --header 'Content-Type: application/json' \\"
+echo " --data '{\"method\":\"debug_traceTransaction\",\"params\":[\"0x...\"],\"id\":1,\"jsonrpc\":\"2.0\"}'"
+echo ""
+echo "📚 DynamoDB Setup Requirements:"
+echo " 1. Create DynamoDB table: dev-erpc-cache (or your environment name)"
+echo " 2. Configure AWS credentials via:"
+echo " - AWS CLI: aws configure"
+echo " - Environment: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY"
+echo " - IAM Role: For EKS/container environments"
+echo ""
+echo "🛑 To stop services: docker-compose down"
\ No newline at end of file
diff --git a/helm/charts/erpc/values.yaml b/helm/charts/erpc/values.yaml
new file mode 100644
index 000000000..3f290978f
--- /dev/null
+++ b/helm/charts/erpc/values.yaml
@@ -0,0 +1,260 @@
+# Default values for erpc
+nameOverride: ""
+fullnameOverride: ""
+
+replicaCount: 3
+
+vault:
+ # Non-secret eRPC config template. Secret placeholders use __SECRET___;
+ # the API-key auth list can be generated from __SECRET_API_KEY_STRATEGIES__.
+ configPath: "secret/data/erpc/config"
+ # Dedicated secret-only entry shared by all eRPC config variants.
+ secretsPath: "secret/data/erpc/secrets"
+ jobImage:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/erpc-validator
+ tag: "0.1.3"
+ pullPolicy: IfNotPresent
+ jobResources:
+ requests:
+ cpu: "50m"
+ memory: "64Mi"
+ limits:
+ cpu: "200m"
+ memory: "256Mi"
+ validationImage:
+ enabled: false
+ repository: ""
+ tag: ""
+ pullPolicy: IfNotPresent
+ command:
+ - erpc
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "128Mi"
+ limits:
+ cpu: "500m"
+ memory: "512Mi"
+
+image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/erpc
+ tag: "0.1.3"
+ pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+
+serviceAccount:
+ # Deprecated/ignored: ServiceAccounts always disable token automount; pods opt in when needed.
+ automount: true
+ # The name of the service account to use.
+ # Legacy creator ServiceAccount name; runtime pods use runtimeName when set.
+ name: ""
+ runtimeName: ""
+ # Set create=false to reuse an existing Vault creator identity while still
+ # allowing the chart to create the runtime ServiceAccount.
+ vaultCreator:
+ # create: false
+ name: ""
+ annotations: {}
+
+# Legacy override hooks (kept for backward compatibility with environments
+# that already set podSecurityContext / securityContext directly on this
+# chart). Active only when securityHardening.enabled is false.
+podSecurityContext: {}
+
+securityContext: {}
+
+# Pod Security Standards "restricted" profile, gated behind an enabled
+# flag so it can be rolled out per-environment. Enable in dev to validate
+# before flipping in prd. When disabled the legacy podSecurityContext /
+# securityContext keys above are honored instead.
+securityHardening:
+ enabled: false
+ podSecurityContext:
+ runAsNonRoot: true
+ runAsUser: 1000
+ fsGroup: 1000
+ seccompProfile:
+ type: RuntimeDefault
+ containerSecurityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop:
+ - ALL
+ readOnlyRootFilesystem: true
+ runAsNonRoot: true
+
+config:
+ server:
+ httpPort: 4000
+ metricsPort: 4001
+
+service:
+ type: ClusterIP
+ port: 4000
+ metricsPort: 4001
+ annotations: {}
+
+metricsService:
+ type: ClusterIP
+ annotations: {}
+
+# Headless service for HAProxy DNS-based pod discovery
+headlessService:
+ enabled: false
+
+autoscaling:
+ enabled: true
+ minReplicas: 3
+ maxReplicas: 8
+ targetCPUUtilizationPercentage: 100
+ targetMemoryUtilizationPercentage: 100
+ behavior:
+ scaleUp:
+ stabilizationWindowSeconds: 120
+ policies:
+ - type: Percent
+ value: 100
+ periodSeconds: 15
+ - type: Pods
+ value: 4
+ periodSeconds: 15
+ selectPolicy: Max
+ scaleDown:
+ stabilizationWindowSeconds: 300
+ policies:
+ - type: Percent
+ value: 50
+ periodSeconds: 30
+ - type: Pods
+ value: 2
+ periodSeconds: 30
+ selectPolicy: Max
+
+resources:
+ requests:
+ cpu: "600m"
+ memory: "1600Mi"
+ limits:
+ cpu: "1000m"
+ memory: "3500Mi"
+
+lifecycle:
+ enabled: true
+ preStop:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - sleep 15
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+extraEnvVars: []
+
+# Startup probe configuration (HTTP /healthcheck on httpPort)
+# Allows slow upstream initialization before liveness/readiness probes start
+# Pod has up to initialDelaySeconds + (periodSeconds * failureThreshold) to start
+startupProbe:
+ path: /healthcheck
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ timeoutSeconds: 5
+ failureThreshold: 6
+
+# Liveness probe configuration (TCP socket on httpPort)
+# Checks if HTTP server is running, otherwise eRPC itself is dead
+# Only runs after startup probe succeeds
+livenessProbe:
+ periodSeconds: 10
+ timeoutSeconds: 5
+ failureThreshold: 3
+
+# Readiness probe configuration (HTTP /healthcheck endpoint)
+# Pod stops receiving traffic if readiness fails
+# Detects when service should be removed from load balancers
+readinessProbe:
+ path: /healthcheck
+ periodSeconds: 5
+ timeoutSeconds: 5
+ failureThreshold: 2
+
+monitoring:
+ podMonitor:
+ enabled: true
+ # Additional labels for the PodMonitor
+ additionalLabels: {}
+ # Annotations for the PodMonitor
+ annotations: {}
+ # Scrape interval
+ interval: 30s
+ # Scrape timeout
+ scrapeTimeout: 10s
+ # Metrics path
+ path: /metrics
+ # Scheme (http or https)
+ scheme: http
+ # Honor labels from the scraped metrics
+ honorLabels: false
+ # Metric relabeling configurations
+ metricRelabelings: []
+ # Relabeling configurations
+ relabelings: []
+ # Namespace selector
+ namespaceSelector: {}
+ # Job label
+ jobLabel: ""
+ # Pod target labels
+ podTargetLabels: []
+ # Sample limit
+ sampleLimit: 0
+ # Target limit
+ targetLimit: 0
+
+postgresql:
+ storage: "500Gi"
+ storageClass: gp3
+ volumeAttributesClassName: ""
+ s3SARoleArn: "arn:aws:iam::537124939463:role/prd-erpc-backup"
+ vault:
+ secretsPath: "secret/data/erpc/db"
+ backup:
+ # Empty = infer volumeSnapshot when volumeSnapshotClassName is set.
+ method: ""
+ volumeSnapshotClassName: ""
+ retentionPolicy: "7d"
+ destinationPath: "s3://prd-morpho-database-backup/erpc-backups"
+ sharedBuffers: "2GB"
+ effectiveCacheSize: "6GB"
+ maintenanceWorkMem: "512MB"
+ workMem: "16MB"
+ maxWorkerProcesses: "2"
+ maxParallelWorkers: "2"
+ maxParallelWorkersPerGather: "1"
+ maxParallelMaintenanceWorkers: "1"
+ autovacuumMaxWorkers: "2"
+ resources:
+ requests:
+ cpu: "2"
+ memory: "8Gi"
+ limits:
+ cpu: "2"
+ memory: "8Gi"
+ # Node affinity for PostgreSQL cluster
+ # Set to null to disable affinity (e.g., in dev)
+ affinity:
+ nodeSelector:
+ purpose: disruptable-database
+ nodegroup: disruptable-database-nodes
+ tolerations:
+ - key: "dedicated"
+ operator: "Equal"
+ value: "disruptable-database"
+ effect: "NoSchedule"
+ # Raw block injected into spec.affinity.additionalPodAntiAffinity. Use to
+ # enforce 1-pod-per-node packing across all CNPG clusters in PRD.
+ additionalPodAntiAffinity: {}
diff --git a/helm/environments/prd/.gitkeep b/helm/environments/prd/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/helm/environments/prd/erpc/Chart.lock b/helm/environments/prd/erpc/Chart.lock
new file mode 100644
index 000000000..ed8c8bf25
--- /dev/null
+++ b/helm/environments/prd/erpc/Chart.lock
@@ -0,0 +1,33 @@
+dependencies:
+- name: erpc
+ repository: file://../../../charts/erpc
+ version: 0.1.35
+- name: erpc
+ repository: file://../../../charts/erpc
+ version: 0.1.35
+- name: erpc
+ repository: file://../../../charts/erpc
+ version: 0.1.35
+- name: erpc
+ repository: file://../../../charts/erpc
+ version: 0.1.35
+- name: erpc
+ repository: file://../../../charts/erpc
+ version: 0.1.35
+- name: redis-ha
+ repository: https://dandydeveloper.github.io/charts
+ version: 4.35.2
+- name: haproxy
+ repository: https://haproxytech.github.io/helm-charts
+ version: 1.26.1
+- name: haproxy
+ repository: https://haproxytech.github.io/helm-charts
+ version: 1.26.1
+- name: haproxy
+ repository: https://haproxytech.github.io/helm-charts
+ version: 1.26.1
+- name: prometheus-redis-exporter
+ repository: https://prometheus-community.github.io/helm-charts
+ version: 6.8.0
+digest: sha256:6f63ffe63edeae52172c50f6d1dd3ed8f12e964a7c8af088c49daeeb4f2a5cda
+generated: "2026-06-16T17:08:14.874547+02:00"
diff --git a/helm/environments/prd/erpc/Chart.yaml b/helm/environments/prd/erpc/Chart.yaml
new file mode 100644
index 000000000..054078259
--- /dev/null
+++ b/helm/environments/prd/erpc/Chart.yaml
@@ -0,0 +1,50 @@
+apiVersion: v2
+name: erpc-prd
+description: eRPC fault-tolerant EVM RPC proxy - Production Environment
+type: application
+version: 0.1.40
+appVersion: "latest"
+
+dependencies:
+ - name: erpc
+ repository: file://../../../charts/erpc
+ version: 0.1.35
+ - name: erpc
+ alias: erpc-dev
+ condition: erpc-dev.enabled
+ repository: file://../../../charts/erpc
+ version: 0.1.35
+ - name: erpc
+ alias: erpc-processing
+ condition: erpc-processing.enabled
+ repository: file://../../../charts/erpc
+ version: 0.1.35
+ - name: erpc
+ alias: erpc-router
+ condition: erpc-router.enabled
+ repository: file://../../../charts/erpc
+ version: 0.1.35
+ - name: erpc
+ alias: erpc-fallback
+ condition: erpc-fallback.enabled
+ repository: file://../../../charts/erpc
+ version: 0.1.35
+ - name: redis-ha
+ version: "4.35.2"
+ repository: "https://dandydeveloper.github.io/charts"
+ - name: haproxy
+ alias: erpc-haproxy
+ repository: https://haproxytech.github.io/helm-charts
+ version: "1.26.1"
+ - name: haproxy
+ alias: erpc-processing-haproxy
+ repository: https://haproxytech.github.io/helm-charts
+ version: "1.26.1"
+ - name: haproxy
+ alias: erpc-router-haproxy
+ repository: https://haproxytech.github.io/helm-charts
+ version: "1.26.1"
+ - name: prometheus-redis-exporter
+ alias: redis-exporter
+ version: "6.8.0"
+ repository: "https://prometheus-community.github.io/helm-charts"
diff --git a/helm/environments/prd/erpc/charts/erpc-0.1.35.tgz b/helm/environments/prd/erpc/charts/erpc-0.1.35.tgz
new file mode 100644
index 000000000..eb0368df3
Binary files /dev/null and b/helm/environments/prd/erpc/charts/erpc-0.1.35.tgz differ
diff --git a/helm/environments/prd/erpc/charts/haproxy-1.26.1.tgz b/helm/environments/prd/erpc/charts/haproxy-1.26.1.tgz
new file mode 100644
index 000000000..1fd3a048a
Binary files /dev/null and b/helm/environments/prd/erpc/charts/haproxy-1.26.1.tgz differ
diff --git a/helm/environments/prd/erpc/charts/prometheus-redis-exporter-6.8.0.tgz b/helm/environments/prd/erpc/charts/prometheus-redis-exporter-6.8.0.tgz
new file mode 100644
index 000000000..20fcaca22
Binary files /dev/null and b/helm/environments/prd/erpc/charts/prometheus-redis-exporter-6.8.0.tgz differ
diff --git a/helm/environments/prd/erpc/charts/redis-ha-4.35.2.tgz b/helm/environments/prd/erpc/charts/redis-ha-4.35.2.tgz
new file mode 100644
index 000000000..9fcf90434
Binary files /dev/null and b/helm/environments/prd/erpc/charts/redis-ha-4.35.2.tgz differ
diff --git a/helm/environments/prd/erpc/templates/_helpers.tpl b/helm/environments/prd/erpc/templates/_helpers.tpl
new file mode 100644
index 000000000..6ff9bbc30
--- /dev/null
+++ b/helm/environments/prd/erpc/templates/_helpers.tpl
@@ -0,0 +1,23 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "erpc-prd.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+*/}}
+{{- define "erpc-prd.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
diff --git a/helm/environments/prd/erpc/templates/erpc-haproxy-hpa.yaml b/helm/environments/prd/erpc/templates/erpc-haproxy-hpa.yaml
new file mode 100644
index 000000000..afceea903
--- /dev/null
+++ b/helm/environments/prd/erpc/templates/erpc-haproxy-hpa.yaml
@@ -0,0 +1,35 @@
+{{- if (index .Values "erpc-haproxy-hpa" "enabled") }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ name: erpc-erpc-haproxy
+ labels:
+ app.kubernetes.io/name: haproxy
+ app.kubernetes.io/instance: erpc-erpc-haproxy
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: erpc-erpc-haproxy
+ minReplicas: {{ index .Values "erpc-haproxy-hpa" "minReplicas" }}
+ maxReplicas: {{ index .Values "erpc-haproxy-hpa" "maxReplicas" }}
+ metrics:
+ {{- if index .Values "erpc-haproxy-hpa" "targetCPUUtilizationPercentage" }}
+ - type: Pods
+ pods:
+ metric:
+ name: cpu_utilization_percentage
+ target:
+ type: AverageValue
+ averageValue: {{ index .Values "erpc-haproxy-hpa" "targetCPUUtilizationPercentage" }}
+ {{- end }}
+ {{- if index .Values "erpc-haproxy-hpa" "targetMemoryUtilizationPercentage" }}
+ - type: Pods
+ pods:
+ metric:
+ name: memory_utilization_percentage
+ target:
+ type: AverageValue
+ averageValue: {{ index .Values "erpc-haproxy-hpa" "targetMemoryUtilizationPercentage" }}
+ {{- end }}
+{{- end }}
diff --git a/helm/environments/prd/erpc/templates/erpc-processing-haproxy-hpa.yaml b/helm/environments/prd/erpc/templates/erpc-processing-haproxy-hpa.yaml
new file mode 100644
index 000000000..0fa8b67e2
--- /dev/null
+++ b/helm/environments/prd/erpc/templates/erpc-processing-haproxy-hpa.yaml
@@ -0,0 +1,35 @@
+{{- if (index .Values "erpc-processing-haproxy-hpa" "enabled") }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ name: erpc-erpc-processing-haproxy
+ labels:
+ app.kubernetes.io/name: haproxy
+ app.kubernetes.io/instance: erpc-erpc-processing-haproxy
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: erpc-erpc-processing-haproxy
+ minReplicas: {{ index .Values "erpc-processing-haproxy-hpa" "minReplicas" }}
+ maxReplicas: {{ index .Values "erpc-processing-haproxy-hpa" "maxReplicas" }}
+ metrics:
+ {{- if index .Values "erpc-processing-haproxy-hpa" "targetCPUUtilizationPercentage" }}
+ - type: Pods
+ pods:
+ metric:
+ name: cpu_utilization_percentage
+ target:
+ type: AverageValue
+ averageValue: {{ index .Values "erpc-processing-haproxy-hpa" "targetCPUUtilizationPercentage" }}
+ {{- end }}
+ {{- if index .Values "erpc-processing-haproxy-hpa" "targetMemoryUtilizationPercentage" }}
+ - type: Pods
+ pods:
+ metric:
+ name: memory_utilization_percentage
+ target:
+ type: AverageValue
+ averageValue: {{ index .Values "erpc-processing-haproxy-hpa" "targetMemoryUtilizationPercentage" }}
+ {{- end }}
+{{- end }}
diff --git a/helm/environments/prd/erpc/templates/erpc-router-haproxy-hpa.yaml b/helm/environments/prd/erpc/templates/erpc-router-haproxy-hpa.yaml
new file mode 100644
index 000000000..8fa9e42fc
--- /dev/null
+++ b/helm/environments/prd/erpc/templates/erpc-router-haproxy-hpa.yaml
@@ -0,0 +1,35 @@
+{{- if (index .Values "erpc-router-haproxy-hpa" "enabled") }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+ name: erpc-erpc-router-haproxy
+ labels:
+ app.kubernetes.io/name: haproxy
+ app.kubernetes.io/instance: erpc-erpc-router-haproxy
+spec:
+ scaleTargetRef:
+ apiVersion: apps/v1
+ kind: Deployment
+ name: erpc-erpc-router-haproxy
+ minReplicas: {{ index .Values "erpc-router-haproxy-hpa" "minReplicas" }}
+ maxReplicas: {{ index .Values "erpc-router-haproxy-hpa" "maxReplicas" }}
+ metrics:
+ {{- if index .Values "erpc-router-haproxy-hpa" "targetCPUUtilizationPercentage" }}
+ - type: Pods
+ pods:
+ metric:
+ name: cpu_utilization_percentage
+ target:
+ type: AverageValue
+ averageValue: {{ index .Values "erpc-router-haproxy-hpa" "targetCPUUtilizationPercentage" }}
+ {{- end }}
+ {{- if index .Values "erpc-router-haproxy-hpa" "targetMemoryUtilizationPercentage" }}
+ - type: Pods
+ pods:
+ metric:
+ name: memory_utilization_percentage
+ target:
+ type: AverageValue
+ averageValue: {{ index .Values "erpc-router-haproxy-hpa" "targetMemoryUtilizationPercentage" }}
+ {{- end }}
+{{- end }}
diff --git a/helm/environments/prd/erpc/templates/job-vault-redis.yaml b/helm/environments/prd/erpc/templates/job-vault-redis.yaml
new file mode 100644
index 000000000..99a760a8f
--- /dev/null
+++ b/helm/environments/prd/erpc/templates/job-vault-redis.yaml
@@ -0,0 +1,100 @@
+{{- if index .Values "redis-ha" "enabled" }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ .Release.Name }}-redis-vault-secret-creator
+ labels:
+ app.kubernetes.io/name: {{ .Release.Name }}-redis-vault
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ annotations:
+ "helm.sh/hook": pre-install,pre-upgrade
+ "helm.sh/hook-weight": "0"
+ "helm.sh/hook-delete-policy": hook-succeeded,hook-failed
+ argocd.argoproj.io/sync-wave: "-1"
+ argocd.argoproj.io/sync-options: Force=true,Replace=true
+spec:
+ backoffLimit: 3
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: {{ .Release.Name }}-redis-vault
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ job: redis-vault-secret-creator
+ annotations:
+ vault.hashicorp.com/agent-inject: "true"
+ vault.hashicorp.com/role: erpc
+ vault.hashicorp.com/agent-inject-secret-redis: {{ index .Values "redis-ha" "vault" "secretsPath" | quote }}
+ vault.hashicorp.com/agent-inject-template-redis: |
+ {{`{{- with secret "`}}{{ index .Values "redis-ha" "vault" "secretsPath" }}{{`" -}}
+ {{- .Data.data.password -}}
+ {{- end }}`}}
+ vault.hashicorp.com/agent-pre-populate-only: "true"
+ spec:
+ serviceAccountName: erpc
+ automountServiceAccountToken: true
+ securityContext:
+ runAsNonRoot: true
+ runAsUser: 1000
+ runAsGroup: 1000
+ fsGroup: 1000
+ seccompProfile:
+ type: RuntimeDefault
+ restartPolicy: OnFailure
+ volumes:
+ - name: tmp
+ emptyDir: {}
+ containers:
+ - name: secret-creator
+ image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/alpine/k8s:1.31.0
+ securityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop:
+ - ALL
+ env:
+ - name: HOME
+ value: /tmp
+ volumeMounts:
+ - name: tmp
+ mountPath: /tmp
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+ limits:
+ cpu: 200m
+ memory: 256Mi
+ command:
+ - sh
+ - -c
+ - |
+ echo "Creating Redis auth secret from Vault..."
+
+ # Wait for vault agent to complete
+ timeout=60
+ count=0
+ while [ ! -f /vault/secrets/redis ] && [ $count -lt $timeout ]; do
+ echo "Waiting for vault secrets file... (${count}s/${timeout}s)"
+ sleep 1
+ count=$((count + 1))
+ done
+
+ if [ ! -f /vault/secrets/redis ]; then
+ echo "ERROR: Vault secrets file not found after $timeout seconds"
+ exit 1
+ fi
+
+ echo "Vault secrets file found. Creating Kubernetes Secret..."
+
+ # Create secret for redis-ha chart (pipe password directly to avoid shell variable exposure)
+ kubectl create secret generic {{ index .Values "redis-ha" "existingSecret" }} \
+ --namespace={{ .Release.Namespace }} \
+ --from-literal={{ index .Values "redis-ha" "authKey" }}="$(cat /vault/secrets/redis)" \
+ --dry-run=client -o yaml | kubectl apply -f -
+
+ echo "Secret {{ index .Values "redis-ha" "existingSecret" }} created successfully"
+
+ # Verify secret was created
+ kubectl get secret {{ index .Values "redis-ha" "existingSecret" }} -n {{ .Release.Namespace }}
+{{- end }}
diff --git a/helm/environments/prd/erpc/templates/prometheus-rules.yaml b/helm/environments/prd/erpc/templates/prometheus-rules.yaml
new file mode 100644
index 000000000..9772e9fce
--- /dev/null
+++ b/helm/environments/prd/erpc/templates/prometheus-rules.yaml
@@ -0,0 +1,335 @@
+{{- $prodErpcSelector := printf "job=~\"%s/erpc(-dev|-processing|-router|-fallback)?\"" .Release.Namespace }}
+{{- $prodErpcMatcher := printf "{%s}" $prodErpcSelector }}
+{{- $prodErpcActionableFailureMatcher := printf "{%s,error!=\"ErrUpstreamsExhausted\"}" $prodErpcSelector }}
+{{- $prodErpcFallbackMatcher := printf "{job=\"%s/erpc-fallback\"}" .Release.Namespace }}
+{{- $prodErpcFallbackActionableFailureMatcher := printf "{job=\"%s/erpc-fallback\",error!=\"ErrUpstreamsExhausted\"}" .Release.Namespace }}
+{{- $prodErpcPrimaryMatcher := printf "{job=~\"%s/erpc(-processing|-router)?\"}" .Release.Namespace }}
+{{- $prodErpcPodRegex := "(erpc|erpc-dev|erpc-processing|erpc-router|erpc-fallback)-[^-]+-[0-9a-z]{5}" }}
+{{- $prodErpcRuntimeContainerRegex := "erpc(-dev|-processing|-router|-fallback)?" }}
+{{- $fullName := include "erpc-prd.fullname" . }}
+{{- $dropLabels := list "agent_name" "container" "container_id" "endpoint" "host_arch" "host_name" "instance" "k8s_pod_name" "namespace" "net_host_name" "net_host_port" "net_peer_name" "net_peer_port" "otel_scope_version" "pod" "process_command" "process_command_args" "process_executable_name" "process_executable_path" "process_owner" "process_pid" "process_runtime_description" "process_runtime_name" "process_runtime_version" "server_address" "server_port" "service_instance_id" | join ", " }}
+{{- $histogramFamilies := list "erpc_upstream_request_duration_seconds" "erpc_network_request_duration_seconds" "erpc_cache_set_success_duration_seconds" "erpc_cache_set_error_duration_seconds" "erpc_network_hedge_delay_seconds" "erpc_network_timeout_duration_seconds" "erpc_network_upstream_calls_per_request" "erpc_cache_get_error_duration_seconds" "erpc_cache_get_success_miss_duration_seconds" "erpc_cache_get_success_hit_duration_seconds" "erpc_network_evm_get_logs_range_requested" "erpc_consensus_agreement_count" "erpc_consensus_responses_collected" "erpc_x402_facilitator_request_duration_seconds" "erpc_multicall3_batch_wait_ms" "erpc_multicall3_batch_size" "erpc_shared_state_poll_lock_wait_duration_seconds" "erpc_shared_state_poll_lock_fallback_duration_seconds" "erpc_http_ingress_pre_forward_duration_seconds" "erpc_rate_limiter_permit_evaluation_duration_seconds" "erpc_rate_limiter_permit_wait_duration_seconds" "erpc_shared_state_poll_lock_hold_duration_seconds" }}
+{{- $counterMetrics := list "erpc_upstream_request_total" "erpc_network_successful_request_total" "erpc_network_request_received_total" "erpc_cache_set_original_bytes_total" "erpc_cache_set_success_total" "erpc_cache_set_error_total" "erpc_cache_set_compressed_bytes_total" "erpc_cache_get_error_total" "erpc_network_evm_block_range_requested_total" "erpc_network_evm_get_logs_forced_splits_total" "erpc_network_evm_get_logs_split_failure_total" "erpc_network_evm_get_logs_split_success_total" "erpc_network_request_self_rate_limited_total" "erpc_upstream_request_remote_rate_limited_total" "erpc_upstream_request_errors_total" "erpc_upstream_request_missing_data_error_total" "erpc_cache_get_success_miss_total" "erpc_cache_get_success_hit_total" "erpc_upstream_finalized_block_polled_total" "erpc_upstream_latest_block_polled_total" "erpc_upstream_stale_upper_bound_total" "erpc_upstream_stale_lower_bound_total" "erpc_upstream_stale_latest_block_total" "erpc_upstream_stale_finalized_block_total" "erpc_upstream_request_empty_response_total" "erpc_network_hedged_request_total" "erpc_network_hedge_discards_total" "erpc_network_multiplexed_request_total" "erpc_multicall3_user_percall_cache_miss_total" "erpc_network_hedge_lost_total" "erpc_network_failed_request_total" "erpc_network_attempt_reason_total" "erpc_cache_get_age_guard_reject_total" "erpc_multicall3_user_percall_cache_hit_total" "erpc_cache_set_skipped_total" "erpc_upstream_sorted_method_cache_pruned_total" "erpc_multicall3_aggregation_total" "erpc_upstream_wrong_empty_response_total" "erpc_multicall3_batch_percall_cache_miss_total" "erpc_cache_envelope_unwrap_failure_total" "erpc_multicall3_batch_percall_cache_set_total" "erpc_shared_state_poll_lock_total" "erpc_multicall3_cache_hits_total" "erpc_multicall3_cache_misses_total" "erpc_cors_disallowed_origin_total" "erpc_cors_preflight_requests_total" "erpc_cors_requests_total" "erpc_consensus_misbehavior_detected_total" "erpc_consensus_total" "erpc_consensus_short_circuit_total" "erpc_consensus_errors_total" "erpc_consensus_cancellations_total" "erpc_rate_limits_total" "erpc_rate_limiter_failopen_total" "erpc_x402_payment_total" "erpc_x402_facilitator_request_total" "erpc_unexpected_panic_total" "erpc_shadow_response_identical_total" "erpc_shadow_response_mismatch_total" "erpc_shadow_response_error_total" }}
+{{- $gaugeMetrics := list "erpc_upstream_latest_block_number" "erpc_upstream_finalized_block_number" "erpc_upstream_block_head_lag" "erpc_upstream_block_head_large_rollback" "erpc_upstream_finalization_lag" "erpc_upstream_routing_priority" "erpc_upstream_score_overall" "erpc_network_latest_block_timestamp_distance_seconds" "erpc_network_dynamic_block_time_milliseconds" "erpc_network_cache_write_queue_depth" "erpc_multicall3_queue_len" "erpc_http_ingress_inflight" "erpc_rate_limiter_budget_max_count" }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+ name: {{ printf "%s-recording-rules" $fullName | trunc 63 | trimSuffix "-" }}
+ labels:
+ helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+ app.kubernetes.io/name: {{ .Chart.Name }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+ app.kubernetes.io/managed-by: {{ .Release.Service }}
+ release: prometheus-stack
+spec:
+ groups:
+ - name: erpc.metric-recording.rules
+ interval: 30s
+ limit: 100000
+ rules:
+ {{- range $family := $histogramFamilies }}
+ {{- range $suffix := list "bucket" "sum" "count" }}
+ - record: "erpc:{{ trimPrefix "erpc_" $family }}_{{ $suffix }}:rate5m"
+ expr: |
+ sum without ({{ $dropLabels }}) (
+ rate({{ $family }}_{{ $suffix }}{{ $prodErpcMatcher }}[5m])
+ )
+ {{- end }}
+ {{- end }}
+ {{- range $metric := $counterMetrics }}
+ - record: "erpc:{{ trimPrefix "erpc_" $metric }}:rate5m"
+ expr: |
+ sum without ({{ $dropLabels }}) (
+ rate({{ $metric }}{{ $prodErpcMatcher }}[5m])
+ )
+ {{- end }}
+ {{- range $metric := $gaugeMetrics }}
+ - record: "erpc:{{ trimPrefix "erpc_" $metric }}:max"
+ expr: |
+ max without ({{ $dropLabels }}) (
+ {{ $metric }}{{ $prodErpcMatcher }}
+ )
+ {{- end }}
+
+ - record: erpc:container_memory_working_set:limit_ratio_by_pod
+ expr: |
+ sum by (pod) (
+ container_memory_working_set_bytes{namespace="{{ .Release.Namespace }}",container=~"{{ $prodErpcRuntimeContainerRegex }}",pod=~"{{ $prodErpcPodRegex }}"}
+ )
+ /
+ clamp_min(
+ sum by (pod) (
+ kube_pod_container_resource_limits{namespace="{{ .Release.Namespace }}",container=~"{{ $prodErpcRuntimeContainerRegex }}",resource="memory",pod=~"{{ $prodErpcPodRegex }}"}
+ ),
+ 1
+ )
+
+ - name: erpc.controller.rules
+ interval: 5s
+ rules:
+ - record: erpc:network_request_duration_seconds_bucket:rate1m_by_pod
+ expr: |
+ sum by (pod, le) (
+ rate(erpc_network_request_duration_seconds_bucket{{ $prodErpcMatcher }}[1m])
+ )
+
+ - record: erpc:network_request_received_total:rate1m_by_pod
+ expr: |
+ sum by (pod) (
+ rate(erpc_network_request_received_total{{ $prodErpcMatcher }}[1m])
+ )
+
+ - record: erpc:network_failed_request_total:rate1m_by_pod
+ expr: |
+ sum by (pod) (
+ rate(erpc_network_failed_request_total{{ $prodErpcMatcher }}[1m])
+ )
+
+ - record: erpc:network_failed_actionable_request_total:rate1m_by_pod
+ expr: |
+ sum by (pod) (
+ rate(erpc_network_failed_request_total{{ $prodErpcActionableFailureMatcher }}[1m])
+ )
+
+ - record: erpc:go_goroutines:by_pod
+ expr: |
+ max by (pod, instance) (
+ go_goroutines{namespace="{{ .Release.Namespace }}",pod=~"{{ $prodErpcPodRegex }}"}
+ )
+
+ - record: erpc:container_cpu_usage:limit_ratio1m_by_pod
+ expr: |
+ sum by (pod) (
+ rate(container_cpu_usage_seconds_total{namespace="{{ .Release.Namespace }}",container=~"{{ $prodErpcRuntimeContainerRegex }}",pod=~"{{ $prodErpcPodRegex }}"}[1m])
+ )
+ /
+ clamp_min(
+ sum by (pod) (
+ kube_pod_container_resource_limits{namespace="{{ .Release.Namespace }}",container=~"{{ $prodErpcRuntimeContainerRegex }}",resource="cpu",pod=~"{{ $prodErpcPodRegex }}"}
+ ),
+ 0.001
+ )
+
+ - name: erpc.fallback-hot-standby.rules
+ interval: 30s
+ rules:
+ - record: erpc:fallback_request_total:rate5m
+ expr: |
+ sum(
+ rate(erpc_network_request_received_total{{ $prodErpcFallbackMatcher }}[5m])
+ )
+
+ - record: erpc:primary_request_total:rate5m
+ expr: |
+ sum(
+ rate(erpc_network_request_received_total{{ $prodErpcPrimaryMatcher }}[5m])
+ )
+
+ - record: erpc:fallback_request_share:ratio5m
+ expr: |
+ (erpc:fallback_request_total:rate5m or on() vector(0))
+ /
+ clamp_min(
+ (erpc:primary_request_total:rate5m or on() vector(0))
+ +
+ (erpc:fallback_request_total:rate5m or on() vector(0)),
+ 1
+ )
+
+ - record: erpc:fallback_failed_request_total:rate5m_by_error_network_method
+ expr: |
+ sum by (error, network, method) (
+ rate(erpc_network_failed_request_total{{ $prodErpcFallbackMatcher }}[5m])
+ )
+
+ - record: erpc:fallback_actionable_failed_request_total:rate5m
+ expr: |
+ sum(
+ rate(erpc_network_failed_request_total{{ $prodErpcFallbackActionableFailureMatcher }}[5m])
+ )
+
+ - record: erpc:fallback_actionable_failure_ratio:ratio5m
+ expr: |
+ (erpc:fallback_actionable_failed_request_total:rate5m or on() vector(0))
+ /
+ clamp_min((erpc:fallback_request_total:rate5m or on() vector(0)), 1)
+
+ - record: erpc:haproxy_request_total:rate5m_by_proxy
+ expr: |
+ sum by (proxy) (
+ rate(haproxy_server_http_responses_total{namespace="{{ .Release.Namespace }}",service=~"erpc-erpc.*haproxy",proxy=~"erpc(-processing|-router)?-(backend|fallback-backend)",server=~"erpc(-fallback-direct|-fallback|-processing|-router)?[0-9]+",code=~"2xx|3xx|4xx|5xx"}[5m])
+ )
+
+ - record: erpc:haproxy_request_total:rate5m_by_proxy_target
+ expr: |
+ sum by (proxy, target) (
+ label_replace(
+ rate(haproxy_server_http_responses_total{namespace="{{ .Release.Namespace }}",service=~"erpc-erpc.*haproxy",proxy=~"erpc(-processing|-router)?-(backend|fallback-backend)",server=~"erpc(-fallback-direct|-fallback|-processing|-router)?[0-9]+",code=~"2xx|3xx|4xx|5xx"}[5m]),
+ "target", "$1", "server", "^(erpc-fallback-direct|erpc-fallback|erpc-processing|erpc-router|erpc)[0-9]+$"
+ )
+ )
+
+ - record: erpc:haproxy_fallback_request_total:rate5m_by_proxy
+ expr: |
+ sum by (proxy) (
+ rate(haproxy_server_http_responses_total{namespace="{{ .Release.Namespace }}",service=~"erpc-erpc.*haproxy",proxy=~"erpc(-processing|-router)?-(backend|fallback-backend)",server=~"erpc-fallback.*",code=~"2xx|3xx|4xx|5xx"}[5m])
+ )
+
+ - record: erpc:haproxy_direct_fallback_request_total:rate5m_by_proxy
+ expr: |
+ sum by (proxy) (
+ rate(haproxy_server_http_responses_total{namespace="{{ .Release.Namespace }}",service=~"erpc-erpc.*haproxy",proxy=~"erpc(-processing|-router)?-fallback-backend",server=~"erpc-fallback-direct[0-9]+",code=~"2xx|3xx|4xx|5xx"}[5m])
+ )
+
+ - record: erpc:haproxy_weighted_fallback_request_total:rate5m_by_proxy
+ expr: |
+ sum by (proxy) (
+ rate(haproxy_server_http_responses_total{namespace="{{ .Release.Namespace }}",service=~"erpc-erpc.*haproxy",proxy=~"erpc(-processing|-router)?-backend",server=~"erpc-fallback[0-9]+",code=~"2xx|3xx|4xx|5xx"}[5m])
+ )
+
+ - record: erpc:haproxy_fallback_request_share:ratio5m_by_proxy
+ expr: |
+ (
+ erpc:haproxy_fallback_request_total:rate5m_by_proxy
+ or on(proxy) (0 * erpc:haproxy_request_total:rate5m_by_proxy)
+ )
+ /
+ clamp_min(erpc:haproxy_request_total:rate5m_by_proxy, 1)
+
+ - alert: ErpcFallbackTrafficHigh
+ expr: |
+ erpc:fallback_request_share:ratio5m > 0.2
+ and
+ erpc:fallback_request_total:rate5m > 10
+ for: 5m
+ labels:
+ severity: warning
+ annotations:
+ summary: "eRPC fallback traffic high"
+ description: "Fallback serves more than 20% of eRPC traffic and more than 10 RPS for 5m. Check main/router/processing health and fallback upstream quota."
+
+ - alert: ErpcFallbackTrafficHighByProxy
+ expr: |
+ erpc:haproxy_fallback_request_share:ratio5m_by_proxy > 0.8
+ and
+ erpc:haproxy_fallback_request_total:rate5m_by_proxy > 25
+ for: 15m
+ labels:
+ severity: warning
+ annotations:
+ summary: "eRPC fallback traffic high for {{`{{ $labels.proxy }}`}}"
+ description: "{{`{{ $labels.proxy }}`}} sends more than 80% of its eRPC traffic and more than 25 RPS to fallback for 15m. Check the matching main/router/processing pool and fallback upstream quota."
+
+ - alert: ErpcFallbackActionableFailuresHigh
+ expr: |
+ erpc:fallback_actionable_failure_ratio:ratio5m > 0.05
+ and
+ erpc:fallback_request_total:rate5m > 10
+ for: 10m
+ labels:
+ severity: warning
+ annotations:
+ summary: "eRPC fallback actionable failures high"
+ description: "Fallback actionable failure ratio is above 5% while fallback serves more than 10 RPS for 10m, excluding ErrUpstreamsExhausted. Check fallback upstream health and quotas."
+
+ - name: erpc.latency.rules
+ interval: 1m
+ rules:
+ - record: erpc:upstream_request_duration_seconds:p50:by_network
+ expr: |
+ histogram_quantile(0.5,
+ sum by (network, le) (erpc:upstream_request_duration_seconds_bucket:rate5m)
+ )
+
+ - record: erpc:upstream_request_duration_seconds:p90:by_network
+ expr: |
+ histogram_quantile(0.9,
+ sum by (network, le) (erpc:upstream_request_duration_seconds_bucket:rate5m)
+ )
+
+ - record: erpc:upstream_request_duration_seconds:p50:by_vendor
+ expr: |
+ histogram_quantile(0.5,
+ sum by (vendor, le) (
+ label_replace(
+ erpc:upstream_request_duration_seconds_bucket:rate5m,
+ "vendor", "$1", "upstream", "([^-]+)-.*"
+ )
+ )
+ )
+
+ - record: erpc:upstream_request_duration_seconds:p90:by_vendor
+ expr: |
+ histogram_quantile(0.9,
+ sum by (vendor, le) (
+ label_replace(
+ erpc:upstream_request_duration_seconds_bucket:rate5m,
+ "vendor", "$1", "upstream", "([^-]+)-.*"
+ )
+ )
+ )
+
+ - record: erpc:upstream_request_duration_seconds:p50
+ expr: |
+ histogram_quantile(0.5,
+ sum by (le) (erpc:upstream_request_duration_seconds_bucket:rate5m)
+ )
+
+ - record: erpc:upstream_request_duration_seconds:p90
+ expr: |
+ histogram_quantile(0.9,
+ sum by (le) (erpc:upstream_request_duration_seconds_bucket:rate5m)
+ )
+
+ - record: erpc:network_request_duration_seconds:p50:by_network
+ expr: |
+ histogram_quantile(0.5,
+ sum by (network, le) (erpc:network_request_duration_seconds_bucket:rate5m)
+ )
+
+ - record: erpc:network_request_duration_seconds:p90:by_network
+ expr: |
+ histogram_quantile(0.9,
+ sum by (network, le) (erpc:network_request_duration_seconds_bucket:rate5m)
+ )
+
+ - record: erpc:cache_get_miss_duration_seconds:p50
+ expr: |
+ histogram_quantile(0.5,
+ sum by (le) (erpc:cache_get_success_miss_duration_seconds_bucket:rate5m)
+ )
+
+ - record: erpc:cache_get_miss_duration_seconds:p90
+ expr: |
+ histogram_quantile(0.9,
+ sum by (le) (erpc:cache_get_success_miss_duration_seconds_bucket:rate5m)
+ )
+
+ - name: erpc.daily.rules
+ interval: 5m
+ rules:
+ - record: erpc:requests:rate5m
+ expr: sum(erpc:network_request_received_total:rate5m)
+
+ - record: erpc:upstream_ratio:rate5m
+ expr: |
+ sum(erpc:upstream_request_total:rate5m)
+ /
+ sum(erpc:network_request_received_total:rate5m)
+
+ - record: erpc:cache_hit_rate:rate5m
+ expr: |
+ sum(erpc:cache_get_success_hit_total:rate5m)
+ /
+ (
+ sum(erpc:cache_get_success_hit_total:rate5m)
+ +
+ sum(erpc:cache_get_success_miss_total:rate5m)
+ )
+
+ - record: erpc:error_rate:rate5m
+ expr: |
+ sum(erpc:upstream_request_errors_total:rate5m)
+ /
+ sum(erpc:upstream_request_total:rate5m)
diff --git a/helm/environments/prd/erpc/templates/rbac-vault-redis.yaml b/helm/environments/prd/erpc/templates/rbac-vault-redis.yaml
new file mode 100644
index 000000000..e55bcebc4
--- /dev/null
+++ b/helm/environments/prd/erpc/templates/rbac-vault-redis.yaml
@@ -0,0 +1,44 @@
+{{- if index .Values "redis-ha" "enabled" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ .Release.Name }}-redis-vault-secret
+ labels:
+ app.kubernetes.io/name: {{ .Release.Name }}-redis-vault
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ annotations:
+ "helm.sh/hook": pre-install,pre-upgrade
+ "helm.sh/hook-weight": "-5"
+ "helm.sh/hook-delete-policy": before-hook-creation
+ argocd.argoproj.io/sync-wave: "-2"
+rules:
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["create"]
+ - apiGroups: [""]
+ resources: ["secrets"]
+ resourceNames:
+ - {{ index .Values "redis-ha" "existingSecret" | quote }}
+ verbs: ["get", "patch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ .Release.Name }}-redis-vault-secret
+ labels:
+ app.kubernetes.io/name: {{ .Release.Name }}-redis-vault
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ annotations:
+ "helm.sh/hook": pre-install,pre-upgrade
+ "helm.sh/hook-weight": "-5"
+ "helm.sh/hook-delete-policy": before-hook-creation
+ argocd.argoproj.io/sync-wave: "-2"
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ .Release.Name }}-redis-vault-secret
+subjects:
+ - kind: ServiceAccount
+ name: erpc
+ namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/helm/environments/prd/erpc/templates/serviceaccount-vault.yaml b/helm/environments/prd/erpc/templates/serviceaccount-vault.yaml
new file mode 100644
index 000000000..93158905c
--- /dev/null
+++ b/helm/environments/prd/erpc/templates/serviceaccount-vault.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: erpc
+ labels:
+ app.kubernetes.io/name: erpc-vault
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ annotations:
+ argocd.argoproj.io/sync-wave: "-4"
+ {{- with .Values.erpc.serviceAccount.annotations }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+automountServiceAccountToken: false
diff --git a/helm/environments/prd/erpc/values.yaml b/helm/environments/prd/erpc/values.yaml
new file mode 100644
index 000000000..3c3b0d59c
--- /dev/null
+++ b/helm/environments/prd/erpc/values.yaml
@@ -0,0 +1,1972 @@
+# =============================================================================
+# HAProxy Smart Load Balancer for eRPC
+# =============================================================================
+# Provides intelligent traffic distribution based on real-time pod metrics:
+# - Goroutines: Direct measure of concurrent work (weight: 30%)
+# - P99 Latency: Response time degradation detection (weight: 35%)
+# - Request Rate: Actual load distribution (weight: 15%)
+# - CPU utilization: Resource pressure indicator (weight: 20%)
+#
+# Traffic flow: Client -> erpc-haproxy:4000 -> erpc pods (via headless service)
+# Auto fallback: erpc-fallback servers stay at weight 0 until Prometheus shows
+# sustained degradation on any public eRPC pool, then the matching HAProxy ramps
+# limited traffic to the upstream-image hot standby pool.
+# =============================================================================
+# Custom HPA for erpc-haproxy (overrides upstream chart HPA which uses metrics-server)
+erpc-haproxy-hpa:
+ enabled: true
+ minReplicas: 3
+ maxReplicas: 100
+ targetCPUUtilizationPercentage: 70
+ targetMemoryUtilizationPercentage: 80
+
+erpc-processing-haproxy-hpa:
+ enabled: true
+ minReplicas: 3
+ maxReplicas: 100
+ targetCPUUtilizationPercentage: 70
+ targetMemoryUtilizationPercentage: 80
+
+erpc-router-haproxy-hpa:
+ enabled: true
+ minReplicas: 3
+ maxReplicas: 100
+ targetCPUUtilizationPercentage: 70
+ targetMemoryUtilizationPercentage: 80
+
+erpc-haproxy:
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/haproxytech/haproxy-alpine
+
+ rbac:
+ create: true
+
+ serviceMonitor:
+ enabled: true
+ extraLabels:
+ release: prometheus
+ endpoints:
+ - port: prometheus
+ path: /metrics
+ scheme: http
+ interval: 15s
+
+ # HAProxy configuration for HTTP RPC load balancing
+ config: |
+ global
+ stats socket /var/run/haproxy/admin.sock mode 660 level admin expose-fd listeners
+ stats timeout 30s
+ maxconn 500000
+ spread-checks 5
+ log stdout format raw local0 info
+ # Raise file descriptor limit to hard limit (was soft=65536, hard=1048576)
+ ulimit-n 1048576
+
+ # Performance tuning - maximized for throughput
+ # tune.bufsize: larger buffer for big JSON-RPC payloads (eth_call responses can be huge)
+ tune.bufsize 131072
+ # tune.maxrewrite: space for header rewriting
+ tune.maxrewrite 8192
+ # tune.http.maxhdr: max headers
+ tune.http.maxhdr 128
+ # tune.recv_enough: receive enough data before processing
+ tune.recv_enough 16384
+ # tune.runqueue-depth: scheduler queue depth for better throughput
+ tune.runqueue-depth 512
+
+ defaults
+ mode http
+ log global
+ option httplog
+ option dontlognull
+ option http-server-close
+ option forwardfor except 127.0.0.0/8
+
+ # Timeouts tuned for blockchain RPC (some calls can be slow)
+ timeout connect 5s
+ timeout client 60s
+ timeout server 60s
+ timeout http-request 30s
+ timeout http-keep-alive 30s
+ # Queue timeout: how long to wait for a server slot
+ timeout queue 120s
+ timeout check 15s
+
+ retries 3
+ option redispatch
+ retry-on conn-failure empty-response response-timeout 503
+
+ # Main frontend - accepts client connections
+ frontend erpc-frontend
+ bind *:4000
+ # Frontend maxconn: match global maxconn (no restriction)
+ maxconn 500000
+
+ # Add request ID for tracing
+ unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid
+ unique-id-header X-Request-ID
+
+ # If every main eRPC server is down, bypass the metric debounce and
+ # route straight to fallback. The controller still handles gradual
+ # degraded-performance activation through erpc-backend weights.
+ use_backend erpc-fallback-backend if { nbsrv(erpc-main-health-backend) eq 0 }
+
+ # Route health through the serving backend so fallback only counts
+ # after activation or total main outage.
+ default_backend erpc-backend
+
+ # Main-only health backend for instant total-outage detection.
+ backend erpc-main-health-backend
+ balance roundrobin
+ option httpchk GET /healthcheck
+ http-check expect status 200
+ default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 check
+ server-template erpc-main-health 100 erpc-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none
+
+ # Main backend - smart load balancing with dynamic weights
+ backend erpc-backend
+ # Weighted round-robin caps fallback traffic by configured weight.
+ # Controller adjusts weights based on Goroutines/Latency/RequestRate/CPU
+ balance roundrobin
+
+ # HTTP health check on application endpoint
+ option httpchk GET /healthcheck
+ http-check expect status 200
+
+ # Connection reuse for efficiency
+ http-reuse safe
+
+ # Aggressive health monitoring
+ # inter=2s: check every 2 seconds
+ # fastinter=1s: check every 1s when transitioning
+ # rise=2: 2 successes to mark healthy
+ # fall=3: 3 failures to mark unhealthy
+ # maxconn=5000: max connections per backend server (100 pods × 5000 = 500K total)
+ default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check
+
+ # DNS-based service discovery via headless service
+ server-template erpc 100 erpc-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none
+ # Weighted fallback is intentionally capped below fallback HPA max so
+ # partial fallback traffic cannot grow with scaled-out fallback pods.
+ server-template erpc-fallback 3 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 weight 0 check resolvers k8s init-addr none
+
+ # Direct fallback backend for sudden total main outage.
+ backend erpc-fallback-backend
+ balance leastconn
+ option httpchk GET /healthcheck
+ http-check expect status 200
+ http-reuse safe
+ default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check
+ server-template erpc-fallback-direct 40 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none
+
+ # Kubernetes DNS resolver
+ resolvers k8s
+ nameserver dns kube-dns.kube-system.svc.cluster.local:53
+ accepted_payload_size 8192
+ hold valid 5s
+ hold nx 5s
+ resolve_retries 3
+ timeout resolve 1s
+ timeout retry 1s
+
+ # Stats/metrics frontend
+ frontend stats
+ mode http
+ bind *:8404
+ http-request use-service prometheus-exporter if { path /metrics }
+ stats enable
+ stats uri /stats
+ stats refresh 10s
+ stats show-legends
+ stats show-node
+
+ # Weight controller sidecar - dynamically adjusts backend weights
+ # Uses 4 metrics: Goroutines, P99 Latency, Request Rate, CPU
+ sidecarContainers:
+ - name: weight-controller
+ image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/haproxy-controller-erpc:v1.2.2
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: PROMETHEUS_URL
+ value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090"
+ # Goroutine query - direct count per pod
+ - name: GOROUTINE_QUERY
+ value: 'erpc:go_goroutines:by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}'
+ # P99 Latency query - in milliseconds
+ - name: LATENCY_QUERY
+ value: 'histogram_quantile(0.99, erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}) * 1000'
+ # Request rate query - requests per second
+ - name: REQUEST_RATE_QUERY
+ value: 'erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}'
+ # CPU query - percentage of limit
+ - name: CPU_QUERY
+ value: '100 * erpc:container_cpu_usage:limit_ratio1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}'
+ # Target thresholds - traffic shifts when exceeded
+ - name: TARGET_GOROUTINES
+ value: "400"
+ - name: TARGET_LATENCY_MS
+ value: "500"
+ - name: TARGET_REQUEST_RATE
+ value: "50"
+ - name: TARGET_CPU
+ value: "60"
+ # Metric weights (sum to 1.0)
+ # Latency has highest weight as it directly measures user impact
+ - name: WEIGHT_GOROUTINES
+ value: "0.30"
+ - name: WEIGHT_LATENCY
+ value: "0.35"
+ - name: WEIGHT_REQUEST_RATE
+ value: "0.15"
+ - name: WEIGHT_CPU
+ value: "0.20"
+ # Backend name must match HAProxy config
+ - name: BACKEND_NAME
+ value: "erpc-backend"
+ # Pod namespace for Kubernetes API lookups
+ - name: POD_NAMESPACE
+ value: "morpho-prd"
+ # Fast updates for responsive load balancing (matches pod scrape)
+ - name: UPDATE_INTERVAL
+ value: "5s"
+ # Weight range
+ - name: MIN_WEIGHT
+ value: "10"
+ - name: MAX_WEIGHT
+ value: "50"
+ - name: HAPROXY_SOCKET
+ value: "/var/run/haproxy/admin.sock"
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: [ALL]
+ readOnlyRootFilesystem: true
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "128Mi"
+ limits:
+ cpu: "500m"
+ memory: "256Mi"
+ livenessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - pgrep -f "controller.sh" > /dev/null
+ initialDelaySeconds: 10
+ periodSeconds: 30
+ failureThreshold: 3
+ readinessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - test -S "$HAPROXY_SOCKET"
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ failureThreshold: 3
+ volumeMounts:
+ - name: haproxy-socket
+ mountPath: /var/run/haproxy
+ - name: controller-tmp
+ mountPath: /tmp
+ - &erpcFallbackController
+ name: fallback-controller
+ image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/haproxy-controller-erpc:v1.2.2
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: PROMETHEUS_URL
+ value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090"
+ - name: BACKEND_NAME
+ value: "erpc-backend"
+ - name: MAIN_HEALTH_BACKEND_NAME
+ value: "erpc-main-health-backend"
+ - name: MAIN_HEALTH_SERVER_PREFIX
+ value: "erpc-main-health"
+ - name: FALLBACK_SERVER_PREFIX
+ value: "erpc-fallback"
+ - name: HAPROXY_SOCKET
+ value: "/var/run/haproxy/admin.sock"
+ - name: UPDATE_INTERVAL_SECONDS
+ value: "30"
+ - name: ENABLE_AFTER_FAILURES
+ value: "6"
+ - name: RECOVER_AFTER_SUCCESSES
+ value: "20"
+ - name: DEGRADED_P99_MS
+ value: "25000"
+ - name: RECOVERED_P99_MS
+ value: "10000"
+ - name: DEGRADED_ERROR_RATE_PERCENT
+ value: "1"
+ - name: RECOVERED_ERROR_RATE_PERCENT
+ value: "0.5"
+ - name: FALLBACK_ACTIVE_WEIGHT
+ value: "1"
+ - name: FALLBACK_IDLE_WEIGHT
+ value: "0"
+ - name: MIN_TRAFFIC_RPS
+ value: "10"
+ - name: REQUEST_RATE_QUERY
+ value: 'sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"})'
+ - name: LATENCY_QUERY
+ value: 'histogram_quantile(0.99, sum by (le) (erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"})) * 1000'
+ - name: ERROR_RATE_QUERY
+ value: '100 * (sum(erpc:network_failed_actionable_request_total:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}) or on() vector(0)) / clamp_min(sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-[0-9a-f]{9,10}-[0-9a-z]{5}"}), 1)'
+ command:
+ - /bin/sh
+ - -ec
+ args:
+ - |
+ log() {
+ echo "$(date '+%Y-%m-%d %H:%M:%S') [$(hostname)] fallback-controller - $*"
+ }
+
+ url_encode() {
+ echo "$1" | sed \
+ -e 's/%/%25/g' \
+ -e 's/ /%20/g' \
+ -e 's/{/%7B/g' \
+ -e 's/}/%7D/g' \
+ -e 's/\[/%5B/g' \
+ -e 's/\]/%5D/g' \
+ -e 's/=/%3D/g' \
+ -e 's/,/%2C/g' \
+ -e 's/>/%3E/g' \
+ -e 's//dev/null || true)"
+
+ if [ -z "$response" ] || echo "$response" | grep -q '"status":"error"'; then
+ log "Prometheus query failed: $query"
+ return 1
+ fi
+
+ value="$(echo "$response" | sed -n 's/.*"value":\[[^,]*,"\([^"]*\)".*/\1/p' | head -1)"
+ case "$value" in
+ ""|NaN|nan|Inf|+Inf|-Inf)
+ log "Prometheus query returned no usable value: $query"
+ return 1
+ ;;
+ esac
+ echo "$value"
+ }
+
+ fallback_servers() {
+ echo "show servers state ${BACKEND_NAME}" | socat stdio "unix-connect:${HAPROXY_SOCKET}" 2>/dev/null | awk -v backend="$BACKEND_NAME" -v prefix="$FALLBACK_SERVER_PREFIX" '
+ $2 == backend && index($4, prefix) == 1 && $5 != "-" { print $4 }
+ '
+ }
+
+ set_fallback_weight() {
+ weight="$1"
+ servers="$(fallback_servers || true)"
+ if [ -z "$servers" ]; then
+ log "No fallback servers currently resolved"
+ return 1
+ fi
+
+ for server in $servers; do
+ command="set server ${BACKEND_NAME}/${server} weight ${weight}"
+ if echo "$command" | socat stdio "unix-connect:${HAPROXY_SOCKET}" >/dev/null 2>&1; then
+ log "Set ${BACKEND_NAME}/${server} weight=${weight}"
+ else
+ log "Failed to set ${BACKEND_NAME}/${server} weight=${weight}"
+ fi
+ done
+ }
+
+ is_gt() {
+ awk -v a="$1" -v b="$2" 'BEGIN { exit !(a > b) }'
+ }
+
+ is_lt() {
+ awk -v a="$1" -v b="$2" 'BEGIN { exit !(a < b) }'
+ }
+
+ fallback_has_active_weight() {
+ echo "show stat" | socat stdio "unix-connect:${HAPROXY_SOCKET}" 2>/dev/null | awk -F, -v backend="$BACKEND_NAME" -v prefix="$FALLBACK_SERVER_PREFIX" -v idle="$FALLBACK_IDLE_WEIGHT" '
+ $1 == backend && index($2, prefix) == 1 && ($19 + 0) > (idle + 0) { found=1 }
+ END { exit !found }
+ '
+ }
+
+ main_health_counts() {
+ stats="$(echo "show stat" | socat stdio "unix-connect:${HAPROXY_SOCKET}" 2>/dev/null)" || return 1
+ echo "$stats" | awk -F, -v backend="$MAIN_HEALTH_BACKEND_NAME" -v prefix="$MAIN_HEALTH_SERVER_PREFIX" '
+ $1 == backend && index($2, prefix) == 1 {
+ if ($18 !~ /^MAINT/) resolved++
+ if ($18 == "UP") up++
+ }
+ END { print (up + 0) ":" (resolved + 0) }
+ '
+ }
+
+ keep_fallback_active_for_direct_outage() {
+ main_health_counts="$(main_health_counts || echo "")"
+ main_healthy_count="${main_health_counts%%:*}"
+ main_resolved_count="${main_health_counts##*:}"
+ if [ -z "$main_health_counts" ] || [ "$main_resolved_count" = "0" ]; then
+ direct_outage_count=0
+ if [ "$main_seen_healthy" -eq 1 ]; then
+ direct_outage_seen=1
+ log "No resolved main eRPC health servers; direct fallback handles outage until main DNS/health recovers"
+ if [ "$active" -eq 1 ]; then
+ set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true
+ fi
+ return 0
+ fi
+ direct_outage_seen=0
+ log "No resolved main eRPC health servers yet; skipping direct-outage fallback activation"
+ return 1
+ fi
+ if [ "$main_healthy_count" = "0" ]; then
+ direct_outage_count=$((direct_outage_count + 1))
+ if [ "$main_seen_healthy" -eq 1 ]; then
+ direct_outage_seen=1
+ fi
+ if [ "$active" -eq 1 ]; then
+ log "No healthy main eRPC servers; preserving weighted fallback recovery handoff"
+ set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true
+ return 0
+ fi
+ if [ "$direct_outage_count" -lt "$ENABLE_AFTER_FAILURES" ]; then
+ log "No healthy main eRPC servers; direct fallback handles outage while weighted recovery handoff waits direct_outage_count=${direct_outage_count}/${ENABLE_AFTER_FAILURES}"
+ return 0
+ fi
+ missing_metrics_count=0
+ degraded_count=0
+ recovered_count=0
+ log "No healthy main eRPC servers for ${direct_outage_count} intervals; keeping direct-outage fallback recovery handoff active"
+ set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true
+ active=1
+ return 0
+ fi
+ main_seen_healthy=1
+ if [ "$direct_outage_seen" -eq 1 ]; then
+ direct_outage_count=0
+ direct_outage_seen=0
+ missing_metrics_count=0
+ degraded_count=0
+ recovered_count=0
+ log "Main eRPC servers recovered after total outage; enabling weighted fallback recovery handoff"
+ set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" && active=1
+ return 0
+ fi
+ direct_outage_count=0
+ return 1
+ }
+
+ sync_startup_state() {
+ if keep_fallback_active_for_direct_outage; then
+ return
+ fi
+
+ startup_request_rate="$(query_prometheus_value "$REQUEST_RATE_QUERY" || echo "")"
+
+ if [ -n "$startup_request_rate" ] && ! is_gt "$startup_request_rate" "$MIN_TRAFFIC_RPS"; then
+ if fallback_has_active_weight; then
+ active=1
+ log "Main eRPC traffic below gate at startup; preserving active fallback weights"
+ else
+ log "Main eRPC traffic below gate at startup; preserving idle fallback weights"
+ fi
+ return
+ fi
+
+ startup_p99_ms="$(query_prometheus_value "$LATENCY_QUERY" || echo "")"
+ startup_error_rate="$(query_prometheus_value "$ERROR_RATE_QUERY" || echo "")"
+
+ if [ -z "$startup_request_rate" ] || [ -z "$startup_p99_ms" ] || [ -z "$startup_error_rate" ]; then
+ if fallback_has_active_weight; then
+ active=1
+ log "Metrics unavailable at startup; preserving active fallback weights"
+ else
+ log "Metrics unavailable at startup; preserving idle fallback weights"
+ fi
+ return
+ fi
+
+ if is_gt "$startup_p99_ms" "$DEGRADED_P99_MS" || is_gt "$startup_error_rate" "$DEGRADED_ERROR_RATE_PERCENT"; then
+ log "Main eRPC already degraded at startup; enabling fallback weight=${FALLBACK_ACTIVE_WEIGHT}"
+ set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" && active=1
+ elif is_lt "$startup_p99_ms" "$RECOVERED_P99_MS" && is_lt "$startup_error_rate" "$RECOVERED_ERROR_RATE_PERCENT"; then
+ log "Main eRPC healthy at startup; disabling fallback weight=${FALLBACK_IDLE_WEIGHT}"
+ set_fallback_weight "${FALLBACK_IDLE_WEIGHT}" && active=0
+ elif fallback_has_active_weight; then
+ active=1
+ log "Startup metrics between thresholds; preserving active fallback weights"
+ else
+ log "Startup metrics between thresholds; preserving idle fallback weights"
+ fi
+ }
+
+ active=0
+ degraded_count=0
+ recovered_count=0
+ missing_metrics_count=0
+ direct_outage_count=0
+ direct_outage_seen=0
+ main_seen_healthy=0
+
+ log "Starting auto fallback controller: traffic gate>${MIN_TRAFFIC_RPS}rps; degrade p99>${DEGRADED_P99_MS}ms or failures>${DEGRADED_ERROR_RATE_PERCENT}% for ${ENABLE_AFTER_FAILURES} intervals; recover p99<${RECOVERED_P99_MS}ms and failures<${RECOVERED_ERROR_RATE_PERCENT}% for ${RECOVER_AFTER_SUCCESSES} intervals"
+ date +%s > /tmp/fallback-controller.heartbeat
+ sync_startup_state || true
+
+ while true; do
+ date +%s > /tmp/fallback-controller.heartbeat
+ if keep_fallback_active_for_direct_outage; then
+ sleep "${UPDATE_INTERVAL_SECONDS}"
+ continue
+ fi
+
+ request_rate="$(query_prometheus_value "$REQUEST_RATE_QUERY" || echo "")"
+
+ if [ -n "$request_rate" ] && ! is_gt "$request_rate" "$MIN_TRAFFIC_RPS"; then
+ missing_metrics_count=0
+ degraded_count=0
+ recovered_count=0
+ log "Traffic below gate; holding fallback state active=${active} request_rate=${request_rate}"
+ if [ "$active" -eq 1 ]; then
+ set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true
+ else
+ set_fallback_weight "${FALLBACK_IDLE_WEIGHT}" || true
+ fi
+ sleep "${UPDATE_INTERVAL_SECONDS}"
+ continue
+ fi
+
+ p99_ms="$(query_prometheus_value "$LATENCY_QUERY" || echo "")"
+ error_rate="$(query_prometheus_value "$ERROR_RATE_QUERY" || echo "")"
+
+ if [ -z "$request_rate" ] || [ -z "$p99_ms" ] || [ -z "$error_rate" ]; then
+ missing_metrics_count=$((missing_metrics_count + 1))
+ degraded_count=0
+ recovered_count=0
+ log "Metrics unavailable; active=${active} missing_metrics_count=${missing_metrics_count}"
+ if [ "$active" -eq 0 ] && [ "$missing_metrics_count" -ge "$ENABLE_AFTER_FAILURES" ]; then
+ log "Main eRPC metrics unavailable; enabling fallback weight=${FALLBACK_ACTIVE_WEIGHT}"
+ set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" && active=1
+ elif [ "$active" -eq 1 ]; then
+ set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true
+ fi
+ sleep "${UPDATE_INTERVAL_SECONDS}"
+ continue
+ fi
+
+ missing_metrics_count=0
+ degraded=0
+ recovered=0
+ if is_gt "$p99_ms" "$DEGRADED_P99_MS" || is_gt "$error_rate" "$DEGRADED_ERROR_RATE_PERCENT"; then
+ degraded=1
+ fi
+ if is_lt "$p99_ms" "$RECOVERED_P99_MS" && is_lt "$error_rate" "$RECOVERED_ERROR_RATE_PERCENT"; then
+ recovered=1
+ fi
+
+ if [ "$degraded" -eq 1 ]; then
+ degraded_count=$((degraded_count + 1))
+ recovered_count=0
+ elif [ "$recovered" -eq 1 ]; then
+ recovered_count=$((recovered_count + 1))
+ degraded_count=0
+ else
+ degraded_count=0
+ recovered_count=0
+ fi
+
+ log "request_rate=${request_rate} p99_ms=${p99_ms} failure_rate_percent=${error_rate} active=${active} degraded_count=${degraded_count} recovered_count=${recovered_count}"
+
+ if [ "$active" -eq 0 ] && [ "$degraded_count" -ge "$ENABLE_AFTER_FAILURES" ]; then
+ log "Main eRPC degraded; enabling fallback weight=${FALLBACK_ACTIVE_WEIGHT}"
+ set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" && active=1
+ elif [ "$active" -eq 1 ] && [ "$recovered_count" -ge "$RECOVER_AFTER_SUCCESSES" ]; then
+ log "Main eRPC recovered; disabling fallback weight=${FALLBACK_IDLE_WEIGHT}"
+ set_fallback_weight "${FALLBACK_IDLE_WEIGHT}" && active=0
+ elif [ "$active" -eq 1 ]; then
+ set_fallback_weight "${FALLBACK_ACTIVE_WEIGHT}" || true
+ else
+ set_fallback_weight "${FALLBACK_IDLE_WEIGHT}" || true
+ fi
+
+ sleep "${UPDATE_INTERVAL_SECONDS}"
+ done
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: [ALL]
+ readOnlyRootFilesystem: true
+ resources:
+ requests:
+ cpu: "50m"
+ memory: "64Mi"
+ limits:
+ cpu: "200m"
+ memory: "128Mi"
+ readinessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - test -S "$HAPROXY_SOCKET"
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ failureThreshold: 3
+ livenessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - test -f /tmp/fallback-controller.heartbeat && [ $(( $(date +%s) - $(cat /tmp/fallback-controller.heartbeat) )) -lt 180 ]
+ initialDelaySeconds: 60
+ periodSeconds: 30
+ failureThreshold: 3
+ volumeMounts:
+ - name: haproxy-socket
+ mountPath: /var/run/haproxy
+ - name: controller-tmp
+ mountPath: /tmp
+
+ extraVolumes:
+ - name: haproxy-socket
+ emptyDir: {}
+ - name: controller-tmp
+ emptyDir: {}
+
+ extraVolumeMounts:
+ - name: haproxy-socket
+ mountPath: /var/run/haproxy
+
+ containerPorts:
+ http: 4000
+ prometheus: 8404
+
+ service:
+ type: ClusterIP
+
+ replicaCount: 3
+
+ resources:
+ requests:
+ cpu: "1"
+ memory: "512Mi"
+ limits:
+ cpu: "4"
+ memory: "2Gi"
+
+ # Disable upstream chart's HPA (uses metrics-server which isn't installed)
+ # Custom HPA is created via erpc-haproxy-hpa values above
+ autoscaling:
+ enabled: false
+
+ # Enable the HAProxy chart's actual PDB key. The previous lower-case
+ # podDisruptionBudget block was ignored by upstream chart 1.26.1.
+ PodDisruptionBudget:
+ enable: true
+ minAvailable: 2
+
+ # Pin the public CloudFront edge proxy to the on-demand node pool. Spot
+ # interruption of a single node previously removed all public HAProxy
+ # replicas at once and caused rpc.morpho.dev CloudFront 504s.
+ nodeSelector:
+ purpose: on-demand
+ tolerations:
+ - key: dedicated
+ operator: Equal
+ value: on-demand
+ effect: NoSchedule
+
+ # Keep the public CloudFront edge proxy off voluntary Karpenter disruption
+ # paths; this protects rpc.morpho.dev while still allowing HPA rollouts.
+ podAnnotations:
+ karpenter.sh/do-not-disrupt: "true"
+
+ podSecurityContext:
+ runAsNonRoot: true
+ runAsUser: 1000
+ fsGroup: 1000
+ seccompProfile:
+ type: RuntimeDefault
+
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: erpc
+ app.kubernetes.io/name: erpc-haproxy
+ topologyKey: kubernetes.io/hostname
+
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: topology.kubernetes.io/zone
+ whenUnsatisfiable: DoNotSchedule
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: erpc
+ app.kubernetes.io/name: erpc-haproxy
+
+erpc-processing-haproxy:
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/haproxytech/haproxy-alpine
+
+ rbac:
+ create: true
+
+ serviceMonitor:
+ enabled: true
+ extraLabels:
+ release: prometheus
+ endpoints:
+ - port: prometheus
+ path: /metrics
+ scheme: http
+
+ config: |
+ global
+ stats socket /var/run/haproxy/admin.sock mode 660 level admin expose-fd listeners
+ stats timeout 30s
+ maxconn 500000
+ spread-checks 5
+ log stdout format raw local0 info
+ ulimit-n 1048576
+ tune.bufsize 131072
+ tune.maxrewrite 8192
+ tune.http.maxhdr 128
+ tune.recv_enough 16384
+ tune.runqueue-depth 512
+
+ defaults
+ mode http
+ log global
+ option httplog
+ option dontlognull
+ option http-server-close
+ option forwardfor except 127.0.0.0/8
+ timeout connect 5s
+ timeout client 60s
+ timeout server 60s
+ timeout http-request 30s
+ timeout http-keep-alive 30s
+ timeout queue 120s
+ timeout check 15s
+ retries 3
+ option redispatch
+ retry-on conn-failure empty-response response-timeout 503
+
+ frontend erpc-processing-frontend
+ bind *:4000
+ maxconn 500000
+ unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid
+ unique-id-header X-Request-ID
+
+ # Direct fallback handles a total processing outage. The default backend
+ # also has metrics-driven weighted fallback for processing-side eRPC degradation.
+ use_backend erpc-processing-fallback-backend if { nbsrv(erpc-processing-main-health-backend) eq 0 }
+
+ default_backend erpc-processing-backend
+
+ backend erpc-processing-main-health-backend
+ balance roundrobin
+ option httpchk GET /healthcheck
+ http-check expect status 200
+ default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 check
+ server-template erpc-processing-main-health 100 erpc-processing-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none
+
+ backend erpc-processing-backend
+ # Weighted round-robin keeps processing pods balanced by the existing
+ # weight-controller and lets fallback stay idle at weight 0 until the
+ # processing-specific fallback-controller activates it.
+ balance roundrobin
+ option httpchk GET /healthcheck
+ http-check expect status 200
+ http-reuse safe
+ default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check
+ server-template erpc-processing 100 erpc-processing-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none
+ # Weighted fallback is intentionally capped below fallback HPA max so
+ # partial fallback traffic cannot grow with scaled-out fallback pods.
+ server-template erpc-fallback 3 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 weight 0 check resolvers k8s init-addr none
+
+ backend erpc-processing-fallback-backend
+ balance leastconn
+ option httpchk GET /healthcheck
+ http-check expect status 200
+ http-reuse safe
+ default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check
+ server-template erpc-fallback-direct 40 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none
+
+ resolvers k8s
+ nameserver dns kube-dns.kube-system.svc.cluster.local:53
+ accepted_payload_size 8192
+ hold valid 5s
+ hold nx 5s
+ resolve_retries 3
+ timeout resolve 1s
+ timeout retry 1s
+
+ frontend stats
+ mode http
+ bind *:8404
+ http-request use-service prometheus-exporter if { path /metrics }
+ stats enable
+ stats uri /stats
+ stats refresh 10s
+ stats show-legends
+ stats show-node
+
+ sidecarContainers:
+ - name: weight-controller
+ image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/haproxy-controller-erpc:v1.2.2
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: PROMETHEUS_URL
+ value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090"
+ - name: GOROUTINE_QUERY
+ value: 'erpc:go_goroutines:by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}'
+ - name: LATENCY_QUERY
+ value: 'histogram_quantile(0.99, erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}) * 1000'
+ - name: REQUEST_RATE_QUERY
+ value: 'erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}'
+ - name: CPU_QUERY
+ value: '100 * erpc:container_cpu_usage:limit_ratio1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}'
+ - name: TARGET_GOROUTINES
+ value: "400"
+ - name: TARGET_LATENCY_MS
+ value: "500"
+ - name: TARGET_REQUEST_RATE
+ value: "50"
+ - name: TARGET_CPU
+ value: "60"
+ - name: WEIGHT_GOROUTINES
+ value: "0.30"
+ - name: WEIGHT_LATENCY
+ value: "0.35"
+ - name: WEIGHT_REQUEST_RATE
+ value: "0.15"
+ - name: WEIGHT_CPU
+ value: "0.20"
+ - name: BACKEND_NAME
+ value: "erpc-processing-backend"
+ - name: POD_NAMESPACE
+ value: "morpho-prd"
+ - name: UPDATE_INTERVAL
+ value: "5s"
+ - name: MIN_WEIGHT
+ value: "10"
+ - name: MAX_WEIGHT
+ value: "50"
+ - name: HAPROXY_SOCKET
+ value: "/var/run/haproxy/admin.sock"
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: [ALL]
+ readOnlyRootFilesystem: true
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "128Mi"
+ limits:
+ cpu: "500m"
+ memory: "256Mi"
+ livenessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - pgrep -f "controller.sh" > /dev/null
+ initialDelaySeconds: 10
+ periodSeconds: 30
+ failureThreshold: 3
+ readinessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - test -S "$HAPROXY_SOCKET"
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ failureThreshold: 3
+ volumeMounts:
+ - name: haproxy-socket
+ mountPath: /var/run/haproxy
+ - name: controller-tmp
+ mountPath: /tmp
+ - <<: *erpcFallbackController
+ env:
+ - name: PROMETHEUS_URL
+ value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090"
+ - name: BACKEND_NAME
+ value: "erpc-processing-backend"
+ - name: MAIN_HEALTH_BACKEND_NAME
+ value: "erpc-processing-main-health-backend"
+ - name: MAIN_HEALTH_SERVER_PREFIX
+ value: "erpc-processing-main-health"
+ - name: FALLBACK_SERVER_PREFIX
+ value: "erpc-fallback"
+ - name: HAPROXY_SOCKET
+ value: "/var/run/haproxy/admin.sock"
+ - name: UPDATE_INTERVAL_SECONDS
+ value: "30"
+ - name: ENABLE_AFTER_FAILURES
+ value: "6"
+ - name: RECOVER_AFTER_SUCCESSES
+ value: "20"
+ - name: DEGRADED_P99_MS
+ value: "25000"
+ - name: RECOVERED_P99_MS
+ value: "5000"
+ - name: DEGRADED_ERROR_RATE_PERCENT
+ value: "2"
+ - name: RECOVERED_ERROR_RATE_PERCENT
+ value: "0.5"
+ - name: FALLBACK_ACTIVE_WEIGHT
+ value: "1"
+ - name: FALLBACK_IDLE_WEIGHT
+ value: "0"
+ - name: MIN_TRAFFIC_RPS
+ value: "50"
+ - name: REQUEST_RATE_QUERY
+ value: 'sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"})'
+ - name: LATENCY_QUERY
+ value: 'histogram_quantile(0.99, sum by (le) (erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"})) * 1000'
+ - name: ERROR_RATE_QUERY
+ value: '100 * (sum(erpc:network_failed_actionable_request_total:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}) or on() vector(0)) / clamp_min(sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-processing-[0-9a-f]{9,10}-[0-9a-z]{5}"}), 1)'
+ extraVolumes:
+ - name: haproxy-socket
+ emptyDir: {}
+ - name: controller-tmp
+ emptyDir: {}
+
+ extraVolumeMounts:
+ - name: haproxy-socket
+ mountPath: /var/run/haproxy
+
+ containerPorts:
+ http: 4000
+ prometheus: 8404
+
+ service:
+ type: ClusterIP
+
+ replicaCount: 3
+
+ resources:
+ requests:
+ cpu: "1"
+ memory: "512Mi"
+ limits:
+ cpu: "4"
+ memory: "2Gi"
+
+ autoscaling:
+ enabled: false
+
+ # Enable the HAProxy chart's actual PDB key. The previous lower-case
+ # podDisruptionBudget block was ignored by upstream chart 1.26.1.
+ PodDisruptionBudget:
+ enable: true
+ minAvailable: 2
+
+ # Pin HAProxy pods to the on-demand node pool so edge/proxy capacity is not
+ # removed by abrupt spot interruptions.
+ nodeSelector:
+ purpose: on-demand
+ tolerations:
+ - key: dedicated
+ operator: Equal
+ value: on-demand
+ effect: NoSchedule
+
+ podSecurityContext:
+ runAsNonRoot: true
+ runAsUser: 1000
+ fsGroup: 1000
+ seccompProfile:
+ type: RuntimeDefault
+
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: erpc
+ app.kubernetes.io/name: erpc-processing-haproxy
+ topologyKey: kubernetes.io/hostname
+
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: topology.kubernetes.io/zone
+ whenUnsatisfiable: DoNotSchedule
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: erpc
+ app.kubernetes.io/name: erpc-processing-haproxy
+
+erpc-router-haproxy:
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/haproxytech/haproxy-alpine
+
+ rbac:
+ create: true
+
+ serviceMonitor:
+ enabled: true
+ extraLabels:
+ release: prometheus
+ endpoints:
+ - port: prometheus
+ path: /metrics
+ scheme: http
+
+ config: |
+ global
+ stats socket /var/run/haproxy/admin.sock mode 660 level admin expose-fd listeners
+ stats timeout 30s
+ maxconn 500000
+ spread-checks 5
+ log stdout format raw local0 info
+ ulimit-n 1048576
+ tune.bufsize 131072
+ tune.maxrewrite 8192
+ tune.http.maxhdr 128
+ tune.recv_enough 16384
+ tune.runqueue-depth 512
+
+ defaults
+ mode http
+ log global
+ option httplog
+ option dontlognull
+ option http-server-close
+ option forwardfor except 127.0.0.0/8
+ timeout connect 5s
+ timeout client 60s
+ timeout server 60s
+ timeout http-request 30s
+ timeout http-keep-alive 30s
+ timeout queue 120s
+ timeout check 15s
+ retries 3
+ option redispatch
+ retry-on conn-failure empty-response response-timeout 503
+
+ frontend erpc-router-frontend
+ bind *:4000
+ maxconn 500000
+ unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid
+ unique-id-header X-Request-ID
+
+ # If every router eRPC server is down, bypass the metric debounce and
+ # route straight to fallback. The controller still handles gradual
+ # degraded-performance activation through erpc-router-backend weights.
+ use_backend erpc-router-fallback-backend if { nbsrv(erpc-router-main-health-backend) eq 0 }
+
+ default_backend erpc-router-backend
+
+ backend erpc-router-main-health-backend
+ balance roundrobin
+ option httpchk GET /healthcheck
+ http-check expect status 200
+ default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 check
+ server-template erpc-router-main-health 100 erpc-router-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none
+
+ backend erpc-router-backend
+ # Weighted round-robin caps fallback traffic by configured weight.
+ balance roundrobin
+ option httpchk GET /healthcheck
+ http-check expect status 200
+ http-reuse safe
+ default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check
+ server-template erpc-router 100 erpc-router-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none
+ # Weighted fallback is intentionally capped below fallback HPA max so
+ # partial fallback traffic cannot grow with scaled-out fallback pods.
+ server-template erpc-fallback 3 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 weight 0 check resolvers k8s init-addr none
+
+ backend erpc-router-fallback-backend
+ balance leastconn
+ option httpchk GET /healthcheck
+ http-check expect status 200
+ http-reuse safe
+ default-server inter 2s fastinter 1s downinter 2s rise 2 fall 3 maxconn 5000 check
+ server-template erpc-fallback-direct 40 erpc-fallback-headless.morpho-prd.svc.cluster.local:4000 check resolvers k8s init-addr none
+
+ resolvers k8s
+ nameserver dns kube-dns.kube-system.svc.cluster.local:53
+ accepted_payload_size 8192
+ hold valid 5s
+ hold nx 5s
+ resolve_retries 3
+ timeout resolve 1s
+ timeout retry 1s
+
+ frontend stats
+ mode http
+ bind *:8404
+ http-request use-service prometheus-exporter if { path /metrics }
+ stats enable
+ stats uri /stats
+ stats refresh 10s
+ stats show-legends
+ stats show-node
+
+ sidecarContainers:
+ - name: weight-controller
+ image: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/morphoorg/haproxy-controller-erpc:v1.2.2
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: PROMETHEUS_URL
+ value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090"
+ - name: GOROUTINE_QUERY
+ value: 'erpc:go_goroutines:by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}'
+ - name: LATENCY_QUERY
+ value: 'histogram_quantile(0.99, erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}) * 1000'
+ - name: REQUEST_RATE_QUERY
+ value: 'erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}'
+ - name: CPU_QUERY
+ value: '100 * erpc:container_cpu_usage:limit_ratio1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}'
+ - name: TARGET_GOROUTINES
+ value: "400"
+ - name: TARGET_LATENCY_MS
+ value: "500"
+ - name: TARGET_REQUEST_RATE
+ value: "50"
+ - name: TARGET_CPU
+ value: "60"
+ - name: WEIGHT_GOROUTINES
+ value: "0.30"
+ - name: WEIGHT_LATENCY
+ value: "0.35"
+ - name: WEIGHT_REQUEST_RATE
+ value: "0.15"
+ - name: WEIGHT_CPU
+ value: "0.20"
+ - name: BACKEND_NAME
+ value: "erpc-router-backend"
+ - name: POD_NAMESPACE
+ value: "morpho-prd"
+ - name: UPDATE_INTERVAL
+ value: "5s"
+ - name: MIN_WEIGHT
+ value: "10"
+ - name: MAX_WEIGHT
+ value: "50"
+ - name: HAPROXY_SOCKET
+ value: "/var/run/haproxy/admin.sock"
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: [ALL]
+ readOnlyRootFilesystem: true
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "128Mi"
+ limits:
+ cpu: "500m"
+ memory: "256Mi"
+ livenessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - pgrep -f "controller.sh" > /dev/null
+ initialDelaySeconds: 10
+ periodSeconds: 30
+ failureThreshold: 3
+ readinessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - test -S "$HAPROXY_SOCKET"
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ failureThreshold: 3
+ volumeMounts:
+ - name: haproxy-socket
+ mountPath: /var/run/haproxy
+ - name: controller-tmp
+ mountPath: /tmp
+ - <<: *erpcFallbackController
+ env:
+ - name: PROMETHEUS_URL
+ value: "http://prometheus-stack-kube-prom-prometheus.monitoring.svc.cluster.local:9090"
+ - name: BACKEND_NAME
+ value: "erpc-router-backend"
+ - name: MAIN_HEALTH_BACKEND_NAME
+ value: "erpc-router-main-health-backend"
+ - name: MAIN_HEALTH_SERVER_PREFIX
+ value: "erpc-router-main-health"
+ - name: FALLBACK_SERVER_PREFIX
+ value: "erpc-fallback"
+ - name: HAPROXY_SOCKET
+ value: "/var/run/haproxy/admin.sock"
+ - name: UPDATE_INTERVAL_SECONDS
+ value: "30"
+ - name: ENABLE_AFTER_FAILURES
+ value: "6"
+ - name: RECOVER_AFTER_SUCCESSES
+ value: "20"
+ - name: DEGRADED_P99_MS
+ value: "25000"
+ - name: RECOVERED_P99_MS
+ value: "10000"
+ - name: DEGRADED_ERROR_RATE_PERCENT
+ value: "1"
+ - name: RECOVERED_ERROR_RATE_PERCENT
+ value: "0.5"
+ - name: FALLBACK_ACTIVE_WEIGHT
+ value: "1"
+ - name: FALLBACK_IDLE_WEIGHT
+ value: "0"
+ - name: MIN_TRAFFIC_RPS
+ value: "10"
+ - name: REQUEST_RATE_QUERY
+ value: 'sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"})'
+ - name: LATENCY_QUERY
+ value: 'histogram_quantile(0.99, sum by (le) (erpc:network_request_duration_seconds_bucket:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"})) * 1000'
+ - name: ERROR_RATE_QUERY
+ value: '100 * (sum(erpc:network_failed_actionable_request_total:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}) or on() vector(0)) / clamp_min(sum(erpc:network_request_received_total:rate1m_by_pod{pod=~"erpc-router-[0-9a-f]{9,10}-[0-9a-z]{5}"}), 1)'
+
+ extraVolumes:
+ - name: haproxy-socket
+ emptyDir: {}
+ - name: controller-tmp
+ emptyDir: {}
+
+ extraVolumeMounts:
+ - name: haproxy-socket
+ mountPath: /var/run/haproxy
+
+ containerPorts:
+ http: 4000
+ prometheus: 8404
+
+ service:
+ type: ClusterIP
+
+ replicaCount: 3
+
+ resources:
+ requests:
+ cpu: "1"
+ memory: "512Mi"
+ limits:
+ cpu: "4"
+ memory: "2Gi"
+
+ autoscaling:
+ enabled: false
+
+ # Enable the HAProxy chart's actual PDB key. The previous lower-case
+ # podDisruptionBudget block was ignored by upstream chart 1.26.1.
+ PodDisruptionBudget:
+ enable: true
+ minAvailable: 2
+
+ # Pin HAProxy pods to the on-demand node pool so edge/proxy capacity is not
+ # removed by abrupt spot interruptions.
+ nodeSelector:
+ purpose: on-demand
+ tolerations:
+ - key: dedicated
+ operator: Equal
+ value: on-demand
+ effect: NoSchedule
+
+ podSecurityContext:
+ runAsNonRoot: true
+ runAsUser: 1000
+ fsGroup: 1000
+ seccompProfile:
+ type: RuntimeDefault
+
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: erpc
+ app.kubernetes.io/name: erpc-router-haproxy
+ topologyKey: kubernetes.io/hostname
+
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: topology.kubernetes.io/zone
+ whenUnsatisfiable: DoNotSchedule
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: erpc
+ app.kubernetes.io/name: erpc-router-haproxy
+
+# =============================================================================
+# eRPC Application Configuration
+# =============================================================================
+erpc:
+ fullnameOverride: "erpc"
+
+ serviceAccount:
+ runtimeName: "erpc-runtime"
+ vaultCreator:
+ create: false
+ name: "erpc"
+
+ # PLA-1360: enable Pod Security Standards "restricted" hardening on the
+ # main erpc prd instance after dev validation. The erpc-dev alias is
+ # already opted in via PR 7536.
+ securityHardening:
+ enabled: true
+
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc
+ tag: &erpcImageTag "0.1.3"
+
+ postgresql:
+ backup:
+ method: volumeSnapshot
+ volumeSnapshotClassName: "csi-aws-vsc"
+ retentionPolicy: "7d"
+ destinationPath: "s3://prd-morpho-database-backup/erpc-backups"
+ # PLA-1112: right-sized from gp3-db-burst (16000/1000) — CloudWatch peak was
+ # only ~339 IOPS / ~8 MB/s, ~50x under burst. gp3-db-standard (3000/125)
+ # leaves ample headroom.
+ storageClass: gp3-db-standard
+ volumeAttributesClassName: gp3-db-standard
+ # Primary currently uses ~693Gi; 500Gi chart default would block replica re-bootstrap
+ storage: "1Ti"
+ # PLA-1112: 8 CPU / 32Gi forced m8g.4xlarge (capped at 8/32 — paying for a
+ # 64Gi/16-vCPU node). 7d peak is only ~0.6 CPU cores and ~0.48Gi RSS.
+ # 7400m/28Gi still rounds up to m8g.4xlarge (both sit at ~98-99% of m8g.2xlarge
+ # allocatable once DaemonSets are counted), so trim to 6800m / 26Gi to land
+ # cleanly on m8g.2xlarge with headroom — half the node, ~26Gi cache for the
+ # ~700GB latency-sensitive DB.
+ resources:
+ requests:
+ cpu: "6800m"
+ memory: "26Gi"
+ limits:
+ cpu: "6800m"
+ memory: "26Gi"
+ # Enforce 1-pod-per-node across all CNPG instances (cross-cluster).
+ affinity:
+ additionalPodAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: cnpg.io/podRole
+ operator: In
+ values: [instance]
+ topologyKey: kubernetes.io/hostname
+ sharedBuffers: "8GB"
+ effectiveCacheSize: "24GB"
+ maintenanceWorkMem: "1GB"
+ workMem: "8MB"
+ maxWorkerProcesses: "4"
+ maxParallelWorkers: "4"
+ maxParallelWorkersPerGather: "2"
+ maxParallelMaintenanceWorkers: "2"
+ autovacuumMaxWorkers: "4"
+
+ vault:
+ jobImage:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator
+ tag: &erpcValidatorImageTag "0.1.3"
+
+ headlessService:
+ enabled: true
+
+ replicaCount: 12
+
+ autoscaling:
+ enabled: true
+ minReplicas: 12
+ maxReplicas: 20
+ targetCPUUtilizationPercentage: 80
+ targetMemoryUtilizationPercentage: 80
+
+ extraEnvVars:
+ - name: GOMEMLIMIT
+ value: "6GiB"
+ - name: GOGC
+ value: "25"
+
+ resources:
+ requests:
+ cpu: "750m"
+ memory: "4Gi"
+ limits:
+ cpu: "1500m"
+ memory: "8Gi"
+
+ startupProbe:
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ timeoutSeconds: 5
+ failureThreshold: 6
+
+ livenessProbe:
+ periodSeconds: 10
+ timeoutSeconds: 5
+ failureThreshold: 3
+
+ readinessProbe:
+ path: /healthcheck
+ periodSeconds: 5
+ timeoutSeconds: 5
+ failureThreshold: 2
+
+ monitoring:
+ podMonitor:
+ enabled: true
+ additionalLabels:
+ release: prometheus
+ interval: 5s
+ scrapeTimeout: 4s
+
+# =============================================================================
+# eRPC Dev Instance - Single pod, no HPA, no database
+# =============================================================================
+erpc-dev:
+ enabled: true
+ fullnameOverride: "erpc-dev"
+
+ # PLA-1360: validate Pod Security Standards "restricted" hardening on the
+ # erpc dev instance (distroless image) before flipping in the other prd
+ # erpc instances.
+ securityHardening:
+ enabled: true
+
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc
+ tag: *erpcImageTag
+
+ replicaCount: 1
+
+ autoscaling:
+ enabled: false
+ minReplicas: 1
+
+ # Disable PostgreSQL (shares the production database via Vault config)
+ postgresql:
+ enabled: false
+
+ # Reuse the existing erpc Vault creator identity only for secret materialization;
+ # runtime pods use erpc-dev-runtime and have no Secret RBAC.
+ serviceAccount:
+ runtimeName: "erpc-dev-runtime"
+ vaultCreator:
+ create: false
+ name: "erpc"
+
+ # Dev config path in Vault. Keep this source cache/state-isolated from prod:
+ # Redis DB/cluster key are dev-only and PostgreSQL cache connector is disabled.
+ vault:
+ configPath: "secret/data/erpc/config-dev"
+ jobImage:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator
+ tag: *erpcValidatorImageTag
+ pullPolicy: IfNotPresent
+
+ # Public ingress for erpc-dev is managed in morpho-infra via CloudFront and
+ # a Terraform-owned internal NLB.
+ service:
+ type: ClusterIP
+
+ extraEnvVars:
+ - name: GOMEMLIMIT
+ value: "640MiB"
+ - name: GOGC
+ value: "50"
+
+ resources:
+ requests:
+ cpu: "250m"
+ memory: "384Mi"
+ limits:
+ cpu: "1200m"
+ memory: "768Mi"
+
+ startupProbe:
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ timeoutSeconds: 5
+ failureThreshold: 6
+
+ livenessProbe:
+ initialDelaySeconds: 30
+ periodSeconds: 10
+ timeoutSeconds: 1
+ failureThreshold: 3
+
+ readinessProbe:
+ path: /healthcheck
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ timeoutSeconds: 5
+ failureThreshold: 2
+
+ monitoring:
+ podMonitor:
+ enabled: true
+ additionalLabels:
+ release: prometheus
+ interval: 5s
+ scrapeTimeout: 4s
+
+# =============================================================================
+# eRPC Fallback Instance - Hot standby upstream-image pool for incident traffic
+# =============================================================================
+erpc-fallback:
+ enabled: true
+ fullnameOverride: "erpc-fallback"
+
+ securityHardening:
+ enabled: true
+
+ image:
+ # Keep the upstream eRPC image for incident fallback, but pin the digest so
+ # pod restarts cannot drift to a new upstream build without review.
+ repository: ghcr.io/erpc/erpc
+ tag: "0.1.0@sha256:f8a292d5e867cf878e5fad1c3b0702531bbf86f3f1d7db1af18e34cbcd5b5a6e"
+ pullPolicy: IfNotPresent
+
+ autoscaling:
+ enabled: true
+ # Keep fallback cheap while idle, but let it burst to full failover capacity
+ # as soon as real traffic or resource pressure appears.
+ minReplicas: 3
+ maxReplicas: 40
+ targetCPUUtilizationPercentage: 50
+ targetMemoryUtilizationPercentage: 70
+ behavior:
+ scaleUp:
+ stabilizationWindowSeconds: 0
+ policies:
+ - type: Percent
+ value: 200
+ periodSeconds: 15
+ - type: Pods
+ value: 16
+ periodSeconds: 15
+ selectPolicy: Max
+ scaleDown:
+ stabilizationWindowSeconds: 300
+ policies:
+ - type: Percent
+ value: 50
+ periodSeconds: 60
+ - type: Pods
+ value: 8
+ periodSeconds: 60
+ selectPolicy: Max
+
+ postgresql:
+ enabled: false
+
+ serviceAccount:
+ runtimeName: "erpc-fallback-runtime"
+ vaultCreator:
+ create: false
+ name: "erpc"
+
+ vault:
+ configPath: "secret/data/erpc/config-fallback"
+ secretsPath: "secret/data/erpc/secrets"
+ jobImage:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator
+ tag: *erpcValidatorImageTag
+ pullPolicy: IfNotPresent
+ validationImage:
+ enabled: true
+ repository: ghcr.io/erpc/erpc
+ tag: "0.1.0@sha256:f8a292d5e867cf878e5fad1c3b0702531bbf86f3f1d7db1af18e34cbcd5b5a6e"
+ pullPolicy: IfNotPresent
+ command:
+ - /erpc-server
+
+ headlessService:
+ enabled: true
+
+ extraEnvVars:
+ - name: GOMEMLIMIT
+ value: "6GiB"
+ - name: GOGC
+ value: "25"
+
+ resources:
+ requests:
+ cpu: "250m"
+ memory: "1Gi"
+ limits:
+ cpu: "1500m"
+ memory: "8Gi"
+
+ lifecycle:
+ enabled: false
+
+ startupProbe:
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ timeoutSeconds: 5
+ failureThreshold: 6
+
+ livenessProbe:
+ initialDelaySeconds: 30
+ periodSeconds: 10
+ timeoutSeconds: 1
+ failureThreshold: 3
+
+ readinessProbe:
+ path: /healthcheck
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ timeoutSeconds: 5
+ failureThreshold: 2
+
+ monitoring:
+ podMonitor:
+ enabled: true
+ additionalLabels:
+ release: prometheus
+ interval: 5s
+ scrapeTimeout: 4s
+
+# =============================================================================
+# eRPC Processing Instance - Dedicated pool for heavier workloads
+# =============================================================================
+erpc-processing:
+ enabled: true
+ fullnameOverride: "erpc-processing"
+
+ # PLA-1360: enable Pod Security Standards "restricted" hardening after
+ # dev validation.
+ securityHardening:
+ enabled: true
+
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc
+ tag: *erpcImageTag
+
+ replicaCount: 3
+
+ autoscaling:
+ enabled: true
+ minReplicas: 3
+ maxReplicas: 8
+ targetCPUUtilizationPercentage: 80
+ targetMemoryUtilizationPercentage: 80
+
+ # Disable PostgreSQL (shares the production database via Vault config)
+ postgresql:
+ enabled: false
+
+ # Reuse the existing erpc Vault creator identity only for secret materialization;
+ # runtime pods use erpc-processing-runtime and have no Secret RBAC.
+ serviceAccount:
+ runtimeName: "erpc-processing-runtime"
+ vaultCreator:
+ create: false
+ name: "erpc"
+
+ # Dedicated processing config path in Vault
+ vault:
+ configPath: "secret/data/erpc/config-processing"
+ jobImage:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator
+ tag: *erpcValidatorImageTag
+ pullPolicy: IfNotPresent
+
+ extraEnvVars:
+ - name: GOMEMLIMIT
+ value: "768MiB"
+ - name: GOGC
+ value: "25"
+
+ resources:
+ requests:
+ cpu: "750m"
+ memory: "512Mi"
+ limits:
+ cpu: "1500m"
+ memory: "1Gi"
+
+ headlessService:
+ enabled: true
+
+ startupProbe:
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ timeoutSeconds: 5
+ failureThreshold: 6
+
+ livenessProbe:
+ periodSeconds: 10
+ timeoutSeconds: 5
+ failureThreshold: 3
+
+ readinessProbe:
+ path: /healthcheck
+ periodSeconds: 5
+ timeoutSeconds: 5
+ failureThreshold: 2
+
+ monitoring:
+ podMonitor:
+ enabled: true
+ additionalLabels:
+ release: prometheus
+ interval: 5s
+ scrapeTimeout: 4s
+
+# =============================================================================
+# eRPC Router Instance - Dedicated pool for router workloads
+# =============================================================================
+erpc-router:
+ enabled: true
+ fullnameOverride: "erpc-router"
+
+ # PLA-1360: enable Pod Security Standards "restricted" hardening after
+ # dev validation.
+ securityHardening:
+ enabled: true
+
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc
+ tag: *erpcImageTag
+
+ replicaCount: 3
+
+ autoscaling:
+ enabled: true
+ minReplicas: 3
+ maxReplicas: 8
+ targetCPUUtilizationPercentage: 80
+ targetMemoryUtilizationPercentage: 80
+
+ # Disable PostgreSQL (shares the production database via Vault config)
+ postgresql:
+ enabled: false
+
+ # Reuse the existing erpc Vault creator identity only for secret materialization;
+ # runtime pods use erpc-router-runtime and have no Secret RBAC.
+ serviceAccount:
+ runtimeName: "erpc-router-runtime"
+ vaultCreator:
+ create: false
+ name: "erpc"
+
+ # Dedicated router config path in Vault
+ vault:
+ configPath: "secret/data/erpc/config-router"
+ jobImage:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/erpc-validator
+ tag: *erpcValidatorImageTag
+ pullPolicy: IfNotPresent
+
+ extraEnvVars:
+ - name: GOMEMLIMIT
+ value: "768MiB"
+ - name: GOGC
+ value: "25"
+
+ resources:
+ requests:
+ cpu: "750m"
+ memory: "512Mi"
+ limits:
+ cpu: "1500m"
+ memory: "1Gi"
+
+ headlessService:
+ enabled: true
+
+ startupProbe:
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ timeoutSeconds: 5
+ failureThreshold: 6
+
+ livenessProbe:
+ periodSeconds: 10
+ timeoutSeconds: 5
+ failureThreshold: 3
+
+ readinessProbe:
+ path: /healthcheck
+ periodSeconds: 5
+ timeoutSeconds: 5
+ failureThreshold: 2
+
+ monitoring:
+ podMonitor:
+ enabled: true
+ additionalLabels:
+ release: prometheus
+ interval: 5s
+ scrapeTimeout: 4s
+
+redis-ha:
+ enabled: true
+ fullnameOverride: "erpc-redis"
+ replicas: 3
+
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/library/redis
+ tag: 8.4.0-alpine
+
+ configmapTest:
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/koalaman/shellcheck
+
+ # Authentication - uses existing secret created from Vault
+ auth: true
+ existingSecret: "erpc-redis-auth"
+ authKey: "password"
+
+ # Vault configuration for Redis password
+ vault:
+ secretsPath: "secret/data/erpc/redis"
+
+ # PodDisruptionBudget - Prevent simultaneous eviction during node scaling
+ podDisruptionBudget:
+ minAvailable: 2 # Always keep at least 2 of 3 Redis pods running
+
+ # Hard anti-affinity ensures pods spread across nodes
+ hardAntiAffinity: true
+
+ # Graceful shutdown - ensure Redis has time to sync before termination
+ terminationGracePeriodSeconds: 60
+
+ # Increase replication backlog for faster resync after restarts
+ redis:
+ # Custom Redis configuration
+ # Larger replication backlog prevents full resync after short disconnections
+ # Default is 1MB, we increase to 256MB to handle longer network partitions
+ config:
+ repl-backlog-size: "256mb"
+ repl-backlog-ttl: "3600"
+ # Reduce replication timeout for faster failover detection
+ repl-timeout: "60"
+ # Ensure RDB persistence for faster recovery
+ save: "900 1 300 10 60 10000"
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "512Mi"
+ limits:
+ cpu: "1"
+ memory: "1Gi"
+ # PreStopHook for graceful failover
+ # Note: The redis-ha chart has a default trigger-failover-if-master.sh script
+ # We add BGSAVE before the failover to ensure data is persisted
+ lifecycle:
+ preStop:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - |
+ # Use REDISCLI_AUTH to avoid password in process args
+ timeout 10 sh -c 'REDISCLI_AUTH="$AUTH" redis-cli BGSAVE' 2>/dev/null || true
+ sleep 2
+ if [ -f /readonly-config/trigger-failover-if-master.sh ]; then
+ /bin/sh /readonly-config/trigger-failover-if-master.sh
+ else
+ echo "Warning: trigger-failover-if-master.sh not found, skipping failover"
+ fi
+
+ haproxy:
+ enabled: true
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/library/haproxy
+ # Enable Prometheus Redis monitoring via annotation-based discovery
+ service:
+ annotations:
+ prometheus.io/redis-scrape: "true"
+ prometheus.io/redis-port: "6379"
+ resources:
+ requests:
+ cpu: "50m"
+ memory: "128Mi"
+ limits:
+ cpu: "500m"
+ memory: "512Mi"
+
+ sentinel:
+ resources:
+ requests:
+ cpu: "10m"
+ memory: "32Mi"
+ limits:
+ cpu: "100m"
+ memory: "64Mi"
+
+ persistentVolume:
+ enabled: true
+ size: 1Gi
+ storageClass: "gp3"
+
+ securityContext:
+ runAsUser: 1000
+ fsGroup: 1000
+ runAsNonRoot: true
+
+ # Explicit on-demand node placement
+ nodeSelector:
+ nodegroup: on-demand-nodes
+
+ tolerations:
+ - key: dedicated
+ operator: Equal
+ value: on-demand
+ effect: NoSchedule
+
+# =============================================================================
+# Redis Exporter for Prometheus monitoring
+# =============================================================================
+redis-exporter:
+ image:
+ repository: 537124939463.dkr.ecr.eu-west-3.amazonaws.com/docker-hub/oliver006/redis_exporter
+
+ # Connect to redis-ha via HAProxy
+ redisAddress: redis://erpc-redis-haproxy:6379
+
+ # Use existing secret created by Vault job
+ auth:
+ enabled: true
+ secret:
+ name: erpc-redis-auth
+ key: password
+
+ # Enable ServiceMonitor for Prometheus
+ serviceMonitor:
+ enabled: true
+ labels:
+ release: prometheus-stack
+ # Add redis_instance label for Grafana dashboard compatibility
+ metricRelabelings:
+ - action: replace
+ sourceLabels: [service]
+ targetLabel: redis_instance
+ regex: (.*)
+ replacement: erpc-redis
+
+ # Resources
+ resources:
+ limits:
+ cpu: 100m
+ memory: 128Mi
+ requests:
+ cpu: 25m
+ memory: 32Mi
diff --git a/review/repo-map.md b/review/repo-map.md
index ab7203c4d..98963b78f 100644
--- a/review/repo-map.md
+++ b/review/repo-map.md
@@ -15,28 +15,25 @@ Top-level
- .semrelrc
- .vscode
- AGENTS.md
-- CLA.md
-- CLAUDE.md
-- CODE_OF_CONDUCT.md
-- CONTRIBUTING.md
-- Dockerfile
-- Dockerfile.erpc-validator
-- Dockerfile.fake
-- Dockerfile.validator
-- LICENSE
-- Makefile
-- README.md
- architecture
- artifacts
- assets
- auth
- biome.json
+- CLA.md
+- CLAUDE.md
- clients
- cmd
+- CODE_OF_CONDUCT.md
- common
- consensus
+- CONTRIBUTING.md
- data
- docker-compose.yml
+- Dockerfile
+- Dockerfile.erpc-validator
+- Dockerfile.fake
+- Dockerfile.validator
- docs
- erpc
- erpc.dist.ts
@@ -46,13 +43,17 @@ Top-level
- go.mod
- go.sum
- health
+- helm
- internal
- kube
+- LICENSE
+- Makefile
- monitoring
- package.json
- pnpm-lock.yaml
- pnpm-workspace.yaml
- prometheus.yaml
+- README.md
- review
- scripts
- scylla
@@ -65,6 +66,7 @@ Top-level
- typescript
- upstream
- util
+- validate-helm-charts.sh
Go entrypoints
- cmd/erpc
diff --git a/scripts/review-repo-map.sh b/scripts/review-repo-map.sh
index 13a7b21b5..ad7cfc1a2 100755
--- a/scripts/review-repo-map.sh
+++ b/scripts/review-repo-map.sh
@@ -19,7 +19,7 @@ echo
echo "Top-level"
if [[ $in_git -eq 1 ]]; then
- git ls-files | awk -F/ '{print $1}' | sort -u | sed 's/^/- /'
+ git ls-files | awk -F/ '{print $1}' | LC_ALL=C sort -f -u | sed 's/^/- /'
else
ls -1 | sed 's/^/- /'
fi
diff --git a/validate-helm-charts.sh b/validate-helm-charts.sh
new file mode 100755
index 000000000..a533802d1
--- /dev/null
+++ b/validate-helm-charts.sh
@@ -0,0 +1,228 @@
+#!/bin/bash
+set -eo pipefail
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Parse command line arguments (PARALLEL_JOBS set after chart discovery)
+PARALLEL_JOBS=""
+while getopts "j:sh" opt; do
+ case $opt in
+ j) PARALLEL_JOBS="$OPTARG" ;;
+ s) SEQUENTIAL=true ;;
+ h)
+ echo "Usage: $0 [-j N] [-s] [-h]"
+ echo " -j N Run N parallel jobs (default: number of charts)"
+ echo " -s Run sequentially (disable parallelism)"
+ echo " -h Show this help"
+ exit 0
+ ;;
+ \?) echo "Invalid option: -$OPTARG" >&2; exit 1 ;;
+ esac
+done
+
+# Check if kubeconform is installed
+if ! command -v kubeconform &> /dev/null; then
+ echo -e "${RED}kubeconform is not installed. Please install it first.${NC}"
+ echo "You can install it via: https://github.com/yannh/kubeconform#installation"
+ exit 1
+fi
+
+# Check if helm is installed
+if ! command -v helm &> /dev/null; then
+ echo -e "${RED}helm is not installed. Please install it first.${NC}"
+ echo "You can install it via: https://helm.sh/docs/intro/install/"
+ exit 1
+fi
+
+# Set base directories to scan (erpc: base chart + prd env wrapper only)
+DIRS_TO_SCAN=("helm/charts/erpc" "helm/environments/prd/erpc")
+
+# List of custom resources to skip
+CUSTOM_RESOURCES="PodMonitor,ServiceMonitor,AlertManager,Prometheus,PrometheusRule,Cluster,ScheduledBackup,ImageCatalog,IngressRoute,Certificate,CronJob,EventSource,EventBus,Sensor"
+export CUSTOM_RESOURCES
+
+# List of charts to explicitly skip validation
+SKIP_VALIDATION_CHARTS=()
+
+# Function to check if a chart should skip validation
+should_skip_validation() {
+ local chart_name=$1
+
+ # Check explicit list
+ for skip_chart in "${SKIP_VALIDATION_CHARTS[@]}"; do
+ if [[ "$chart_name" == "$skip_chart" ]]; then
+ return 0 # true, should skip
+ fi
+ done
+
+ # Check database pattern
+ if [[ "$chart_name" == *"-db" || "$chart_name" == *"database"* || "$chart_name" == *"postgres"* ]]; then
+ return 0 # true, should skip
+ fi
+
+ return 1 # false, should not skip
+}
+
+# Export for parallel subshells
+export -f should_skip_validation
+export SKIP_VALIDATION_CHARTS
+
+# Function to process a single chart (for parallel execution)
+process_chart() {
+ local chart_path=$1
+ local chart_name=$(basename "$chart_path")
+ local result_dir=$(mktemp -d)
+ local lint_file="$result_dir/lint"
+ local template_file="$result_dir/template"
+ local has_error=0
+
+ # Store the absolute path to the chart
+ local chart_absolute_path=$(realpath "$chart_path")
+
+ echo "Processing: ${chart_path}" >&2
+
+ # Run helm lint
+ if ! helm lint "$chart_absolute_path" > "$lint_file" 2>&1; then
+ echo -e "${RED}✗ Helm lint failed for ${chart_name}${NC}" >&2
+ cat "$lint_file" >&2
+ has_error=1
+ else
+ echo -e "${GREEN}✓ Helm lint passed for ${chart_name}${NC}" >&2
+ fi
+
+ # Check if this chart should skip validation
+ if should_skip_validation "$chart_name"; then
+ echo -e "${YELLOW}⊘ Skipping kubeconform for ${chart_name} (database chart)${NC}" >&2
+ rm -rf "$result_dir"
+ return $has_error
+ fi
+
+ # Build dependencies if Chart.yaml exists and has dependencies
+ if [ -f "${chart_absolute_path}/Chart.yaml" ]; then
+ if grep -q "dependencies:" "${chart_absolute_path}/Chart.yaml"; then
+ if ! helm dependency update "$chart_absolute_path" > /dev/null 2>&1; then
+ echo -e "${RED}✗ Dependency update failed for ${chart_name}${NC}" >&2
+ rm -rf "$result_dir"
+ return 1
+ fi
+ fi
+
+ # Run template validation: render the chart and validate the
+ # resulting manifests with kubeconform. The pipeline's exit status
+ # reflects kubeconform (the last command), which is what we check.
+ if ! helm template "$chart_absolute_path" 2>/dev/null \
+ | kubeconform --ignore-missing-schemas --summary --skip "$CUSTOM_RESOURCES" \
+ > "$template_file" 2>&1; then
+ echo -e "${RED}✗ Kubeconform failed for ${chart_name}${NC}" >&2
+ cat "$template_file" >&2
+ has_error=1
+ else
+ echo -e "${GREEN}✓ Kubeconform passed for ${chart_name}${NC}" >&2
+ fi
+ else
+ echo -e "${RED}No Chart.yaml found in ${chart_path}${NC}" >&2
+ has_error=1
+ fi
+
+ rm -rf "$result_dir"
+ return $has_error
+}
+
+# Export for parallel
+export -f process_chart
+
+# Main script execution
+echo -e "${YELLOW}Starting Helm chart validation...${NC}"
+echo -e "${YELLOW}Skipping validation for these custom resources: ${CUSTOM_RESOURCES}${NC}"
+
+# Add common repositories that might be needed
+echo -e "${YELLOW}Adding common Helm repositories...${NC}"
+helm repo add bitnami https://charts.bitnami.com/bitnami 2>/dev/null || true
+helm repo add cnpg https://cloudnative-pg.github.io/charts 2>/dev/null || true
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 2>/dev/null || true
+helm repo add haproxytech https://haproxytech.github.io/helm-charts 2>/dev/null || true
+helm repo add dandydeveloper https://dandydeveloper.github.io/charts 2>/dev/null || true
+
+# Update Helm repositories
+echo -e "${YELLOW}Updating Helm repositories...${NC}"
+helm repo update
+
+# Collect all charts to validate
+CHARTS=()
+for dir in "${DIRS_TO_SCAN[@]}"; do
+ if [ ! -d "$dir" ]; then
+ echo -e "${YELLOW}Directory $dir does not exist, skipping...${NC}"
+ continue
+ fi
+
+ while IFS= read -r chart_yaml; do
+ chart_dir=$(dirname "$chart_yaml")
+ CHARTS+=("$chart_dir")
+ done < <(find "$dir" -name Chart.yaml -type f)
+done
+
+TOTAL_CHARTS=${#CHARTS[@]}
+
+# Set PARALLEL_JOBS to number of charts if not specified
+if [ -z "$PARALLEL_JOBS" ]; then
+ PARALLEL_JOBS=$TOTAL_CHARTS
+fi
+
+echo -e "\n${YELLOW}Found ${TOTAL_CHARTS} charts to validate${NC}\n"
+
+# Run validation
+FAILED_CHARTS=()
+
+if [ "$SEQUENTIAL" = true ] || ! command -v parallel &> /dev/null; then
+ # Sequential execution (fallback or explicit)
+ if [ "$SEQUENTIAL" != true ]; then
+ echo -e "${YELLOW}GNU parallel not found, falling back to sequential execution${NC}"
+ echo -e "${YELLOW}Install with: brew install parallel (macOS) or apt install parallel (Linux)${NC}\n"
+ fi
+
+ for chart in "${CHARTS[@]}"; do
+ if ! process_chart "$chart"; then
+ FAILED_CHARTS+=("$chart")
+ fi
+ done
+else
+ # Parallel execution
+ echo -e "${BLUE}Running ${PARALLEL_JOBS} parallel validation jobs...${NC}\n"
+
+ # Create a temporary file to track failures
+ FAIL_FILE=$(mktemp)
+
+ # Run in parallel, collecting failures
+ printf '%s\n' "${CHARTS[@]}" | \
+ parallel -j "$PARALLEL_JOBS" --halt never,fail=1 --line-buffer \
+ "process_chart {} || echo {} >> $FAIL_FILE"
+
+ # Collect failed charts
+ if [ -f "$FAIL_FILE" ]; then
+ while IFS= read -r failed; do
+ FAILED_CHARTS+=("$failed")
+ done < "$FAIL_FILE"
+ rm -f "$FAIL_FILE"
+ fi
+fi
+
+# Summary report
+echo -e "\n${YELLOW}==== Validation Summary ====${NC}"
+echo -e "Total charts processed: ${TOTAL_CHARTS}"
+echo -e "Failed charts: ${#FAILED_CHARTS[@]}"
+
+if [ ${#FAILED_CHARTS[@]} -eq 0 ]; then
+ echo -e "${GREEN}All validations passed successfully!${NC}"
+ exit 0
+else
+ echo -e "${RED}The following charts failed validation:${NC}"
+ for chart in "${FAILED_CHARTS[@]}"; do
+ echo -e " - ${chart}"
+ done
+ exit 1
+fi