added repo

helm-templates/victoria-metrics-alerts-config/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm-templates/victoria-metrics-alerts-config/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v2
+name: victoria-metrics-alerts-config
+description: Generic alerts config and for all BUs
+version: 1.0.0

helm-templates/victoria-metrics-alerts-config/Jenkinsfile

@@ -0,0 +1,19 @@
+@Library('devops-lib@alert-setup') _
+
+properties([
+    parameters([
+        string(defaultValue: '', name: 'app_repo_name', description: 'Repository name of the application. Example: supplier-payment, ads-credit etc', trim: true),
+        string(defaultValue: 'main', name: 'app_repo_branch_name', description: 'Branch name of above provided repo. Example: master, develop', trim: true),
+        string(defaultValue: '', name: 'service', description: 'K8s Service/app name. Example: supplier-payment-web, ads-credit ', trim: true),
+        choice(name: 'ingress', choices:'contour\nnginx', description: 'Type of ingress controller'),
+        choice(choices: ['critical', 'warning'], description: 'NOTE: This is required', name: 'severity'),
+        string(defaultValue: '', name: 'alert_4xx_value', description: 'Threshold value of 4XX Error Count Rate for alerting. Example: 100, 200, 0.75 etc', trim: true),
+        string(defaultValue: '', name: 'alert_5xx_value', description: 'Threshold value of 5XX Error Count Rate for alerting. Example: 100, 200, 0.75 etc', trim: true),
+        string(defaultValue: '', name: 'alert_p99_value', description: 'Threshold value of P99 Latency in MILI SECONDS for alerting. \n NOTE: Unit = MILI SECONDS', trim: true),
+        choice(choices: ['1m', '2m', '3m','5m', '10m'], description: 'NOTE: This is required. Default - 1m', name: 'alert_4xx_period'),
+        choice(choices: ['1m', '2m', '3m','5m', '10m'], description: 'NOTE: This is required. Default - 1m', name: 'alert_5xx_period'),
+        choice(choices: ['1m', '2m', '3m','5m', '10m'], description: 'NOTE: This is required. Default - 1m', name: 'alert_p99_period')
+    ])
+])
+
+createAlerts(params)

helm-templates/victoria-metrics-alerts-config/Jenkinsfile_cron

@@ -0,0 +1,233 @@
+@Library('terraform-infra@main') _
+
+pipeline {
+    agent {
+        node {
+            label 'slave-01'
+        }
+    }
+
+    options {
+        timestamps()
+        ansiColor('xterm')
+    }
+
+    environment {
+        GITHUB_CRED = 'meesho-jenkins'
+    }
+
+    stages {
+        stage('Setup parameters') {
+            steps {
+                script {
+                    properties([
+                        parameters([
+                            string(
+                                defaultValue: '',
+                                name: 'repo_name',
+                                description: 'Repository name of the application. Example: supplier-payment, ads-credit etc',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: 'main',
+                                name: 'branch_name',
+                                description: 'Branch name of above provided repo. Example: master, develop',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'service',
+                                description: 'K8s Service/app name. Example: supplier-payment-web, ads-credit ',
+                                trim: true
+                            ),
+                            choice(
+                                choices: ['critical', 'warning'],
+                                description: 'NOTE: This is required',
+                                name: 'severity'
+                            ),
+                            choice(
+                                choices: ['prd', 'ftr', 'dev', 'int'],
+                                description: 'NOTE: This is required',
+                                name: 'environment'
+                            ),
+                            choice(
+                                choices: ['1m', '2m', '5m', '10m', '1h', '2h'],
+                                description: 'NOTE: This is required. Default - 1m',
+                                name: 'period'
+                            )
+                        ])
+                    ])
+                }
+            }
+        }
+
+        stage('validation') {
+            steps {
+                script {
+                    env.msg = 'Started service alert onboarding'
+                    env.metric = 'CronJob_Alert'
+                    env.int_env = 'main'
+                    if (period.contains('m')) {
+                        env.threshold = period.replaceAll('m', '').toInteger() * 60
+                    }
+                    else if (period.contains('h')) {
+                        env.threshold = period.replaceAll('h', '').toInteger() * 60 * 60
+                    }
+                    else {
+                        env.threshold = null
+                    }
+                    log.info("""
+                        App Repo Name - ${repo_name}
+                        App Repo Branch Name - ${branch_name}
+                        Threshold - ${env.threshold}
+                        Service Name - ${service}
+                        Severity - ${severity}
+                        Env - ${environment}
+                        Period - ${period}
+                    """)
+                    if (params.repo_name.isEmpty() || params.branch_name.isEmpty() || params.service.isEmpty()) {
+                        env.msg += '\n\nFAILED -\n  One or more input paramters are EMPTY/NULL.'
+                        log.err("${env.msg}")
+                    }
+                }
+            }
+        }
+
+        stage('Checkout Terraform-Infra repo') {
+            steps {
+                script {
+                    log.info('Checkout Terraform-Infra repo')
+                    gitActions.checkout()
+                    env.msg += '\n2. Terraform-Infra Repo checkout step was SUCCESSFUL.'
+                    // gitActions.branchCheckOut("${WORKSPACE}/${repo_name}", "${branch_name}")
+                }
+            }
+        }
+
+        stage('Clone service repo for fetching app details') {
+            steps {
+                script {
+                    log.info("Cloning App Repo - ${repo_name}, Branch - ${branch_name}")
+                    gitActions.clone("${WORKSPACE}", "${repo_name}", "${branch_name}")
+                    // sh 'ls -al'
+                    env.msg += '\n3. Service Repo Clone step was SUCCESSFUL.'
+                    // gitActions.branchCheckOut("${WORKSPACE}/${repo_name}", "${branch_name}")
+                }
+            }
+        }
+
+        stage('Collecting app details') {
+            steps {
+                script {
+                    dir("${repo_name}/deployments/") {
+                        sh 'ls -al'
+                        try {
+                            def i = 0
+                            log.info("Reading BU, Team etc from ${repo_name}/deployments/${service}.yaml")
+                            cron = sh(returnStdout: true, script: "yq '.cron' ${service}.yaml").trim()
+                            if (!cron.toBoolean()) {
+                                env.msg += '\nProvided service is not a Scheduler/Cron Type.'
+                                sh 'exit 1'
+                            }
+                            bu = sh(returnStdout: true, script: "yq '.bu' ${service}.yaml").trim()
+                            team = sh(returnStdout: true, script: "yq '.team' ${service}.yaml").trim()
+                            priority = sh(returnStdout: true, script: "yq '.priority' ${service}.yaml").trim()
+                            cronjobs = sh(returnStdout: true, script: "yq '.environment.${environment}.jobs | keys' ${service}.yaml | sed 's/- //g' | tail -n 1").trim()
+                            echo "${cronjobs}"
+                            while (i <= cronjobs.toInteger()) {
+                                cronjob_name = sh(returnStdout: true, script: "yq '.environment.${environment}.jobs[${i}] | keys' ${service}.yaml | sed 's/- //g'").trim()                                
+                                echo "${cronjob_name}"
+                                cronjob_name = cronjob_name.replaceAll('-', '_')
+                                log.info("Creating alert-config.yaml for ${i}. ${cronjob_name}")
+                                create_config()
+                                i += 1
+                            }
+                        }
+                        catch (Exception e) {
+                            env.msg += "\n\nFAILED -\n  Error while fetching app details from the app repo provided.\n  Full Error Details - ${e}"
+                            log.err("${env.msg}")
+                        }
+                    }
+                }
+            }
+        }
+
+        // stage('Create alert-config.yaml') {
+        //     steps {
+        //         script {
+        //             try {
+        //                 log.info('Creating alert-config.yaml')
+        //                 create_config()
+        //                 env.msg += '\n4. Create alert-config.yaml step was SUCCESSFUL.'
+        //             }
+        //             catch (Exception e) {
+        //                 env.msg += "\n\nFAILED -\n  Error while creating the alert-config.yaml.\n  Full Error Details - ${e}"
+        //                 log.err("${env.msg}")
+        //             }
+        //         }
+        //     }
+        // }
+
+        stage('Commit alert-config.yaml') {
+            steps {
+                script {
+                    new_branch_name = "${bu}-${team}-${service}"
+                    log.info("Pushing alert-config.yaml to Branch - ${new_branch_name}")
+                    // log.info("Pushing alert-config.yaml to Branch - ${service}")
+                    sh "rm -rf ${repo_name}"
+                    gitActions.branchCheckOut("${WORKSPACE}", "${new_branch_name}")
+                    // gitActions.branchCheckOut("${WORKSPACE}", "${service}")
+                    gitActions.add("${WORKSPACE}", '.')
+                    gitActions.codeCommit("${WORKSPACE}", "${new_branch_name}", "Updated ${env.metric} for ${new_branch_name}")
+                    gitActions.codePush("${WORKSPACE}", "${new_branch_name}")
+                    // gitActions.codeCommit("${WORKSPACE}", "${service}", 'Updated alert-config.yaml')
+                    // gitActions.codePush("${WORKSPACE}", "${service}")
+                    env.msg += '\n5. Pushing alert-config.yaml step is SUCCESSFUL. Please make sure to delete branch after PR Merge.'
+                }
+            }
+        }
+
+        stage('Raising PR') {
+            steps {
+                script {
+                    log.info("Creating PR -> Base - main; Head - ${new_branch_name}")
+                    pr_message = "Onboarding Alerts for ${service}"
+                    gitActions.createPR("${WORKSPACE}", 'terraform-infra', "${new_branch_name}", "${pr_message}")
+                    env.msg += "\n6. Creating PR, Head - ${new_branch_name} and Base - main is SUCCESSFUL."
+                }
+            }
+        }
+    }
+    post {
+        always {
+            cleanWs()
+        }
+    }
+}
+
+def create_config() {
+    dir("${WORKSPACE}/meesho/tools/vm-alerts-config/configmap") {
+        sh 'pwd'
+        sh "mkdir -p ${bu}/${team}/${service}/mandatory_alerts"
+        sh "touch ${bu}/${team}/${service}/mandatory_alerts/${cronjob_name}-${env.metric}.yaml"
+
+        def config_template = libraryResource 'com/meesho/template-cronjob-alert-configmap.yaml'
+        def config_binding = []
+
+        config_binding = [
+            'threshold': "${env.threshold}",
+            'cronjob_name': "${cronjob_name}",
+            'period': "${period}",
+            'severity': "${severity}",
+            'bu': "${bu}",
+            'team': "${team}",
+            'service': "${service}",
+            'environment': "${environment}",
+            'priority': "${priority}",
+        ]
+        writeFile file:"${bu}/${team}/${service}/mandatory_alerts/${cronjob_name}-${env.metric}.yaml", text: tokenize(config_template, config_binding)
+
+        sh "cat ${bu}/${team}/${service}/mandatory_alerts/${cronjob_name}-${env.metric}.yaml"
+        sh "pwd;ls -al ${bu}/${team}/${service}/mandatory_alerts;ls -al .."
+    }
+}

helm-templates/victoria-metrics-alerts-config/Jenkinsfile_migration

@@ -0,0 +1,248 @@
+@Library('terraform-infra@main') _
+
+pipeline {
+    agent {
+        node {
+            label 'slave-01'
+        }
+    }
+
+    options {
+        timestamps()
+        ansiColor('xterm')
+    }
+
+    environment {
+        GITHUB_CRED = 'meesho-jenkins'
+    }
+
+    stages {
+        stage('Setup parameters') {
+            steps {
+                script {
+                    properties([
+                        parameters([
+                            string(
+                                defaultValue: '',
+                                name: 'repo_name',
+                                description: 'Repository name of the application. Example: supplier-payment, ads-credit etc',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: 'main',
+                                name: 'branch_name',
+                                description: 'Branch name of above provided repo. Example: master, develop',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'service',
+                                description: 'K8s Service/app name. Example: supplier-payment-web, ads-credit ',
+                                trim: true
+                            ),
+                            choice(
+                                choices: ['critical', 'warning'],
+                                description: 'NOTE: This is required',
+                                name: 'severity'
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'alert_name',
+                                description: 'Metric to be onbaorded to Alertmanager. Please use capital letters and underscore (_) only and don’t use spaces ( ). Example: HTTP_4XX_Alert_Per_API',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'query',
+                                description: 'Copy the PromQL Query from existing Grafana Dashboard and paste it here',
+                                trim: true
+                            ),
+                            choice(
+                                choices: ['>=', '>', '=', '<=', '<'],
+                                description: 'NOTE: This is required',
+                                name: 'comparison_operator'
+                            ),
+                            choice(
+                                choices: ['prd', 'ftr', 'dev', 'int'],
+                                description: 'NOTE: This is required',
+                                name: 'environment'
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'threshold',
+                                description: 'Threshold value of alerting. Example: 100, 200, 0.75 etc',
+                                trim: true
+                            ),
+                            choice(
+                                choices: ['1m', '2m', '5m', '10m'],
+                                description: 'NOTE: This is required. Default - 1m',
+                                name: 'period'
+                            )
+                        ])
+                    ])
+                }
+            }
+        }
+
+        stage('validation') {
+            steps {
+                script {
+                    env.msg = 'Started service alert onboarding'
+                    alert_name = alert_name.replaceAll("${service}", '')
+                    alert_name = alert_name.replaceAll(' ', '_')
+                    alert_name = alert_name.replaceAll('-', '_')
+                    alert_name = alert_name.replaceAll('__', '_')
+                    def metric_details = [
+                            'alert_name': "${alert_name}_EKS",
+                            'summary': "EKS ${alert_name} on {{ \$labels.service }}"
+                    ]
+                    env.alert_name = metric_details['alert_name']
+                    env.summary = metric_details['summary']
+                    env.int_env = 'main'
+
+                    log.info("""
+                        App Repo Name - ${repo_name}
+                        App Repo Branch Name - ${branch_name}
+                        Threshold - ${threshold}
+                        Service Name - ${service}
+                        Severity - ${severity}
+                        Comparison Operator - ${comparison_operator}
+                        Env - ${environment}
+                        Period - ${period}
+                        Alert Name - ${env.alert_name}
+                        Query - ${query}
+                        Summary - ${env.summary}
+                    """)
+                    if (params.repo_name.isEmpty() || params.branch_name.isEmpty() || params.threshold.isEmpty() || params.service.isEmpty()) {
+                        env.msg += '\n\nFAILED -\n  One or more input paramters are EMPTY/NULL.'
+                        log.err("${env.msg}")
+                    }
+                }
+            }
+        }
+
+        stage('Checkout Terraform-Infra repo') {
+            steps {
+                script {
+                    log.info('Checkout Terraform-Infra repo')
+                    gitActions.checkout()
+                    env.msg += '\n2. Terraform-Infra Repo checkout step was SUCCESSFUL.'
+                    // gitActions.branchCheckOut("${WORKSPACE}/${repo_name}", "${branch_name}")
+                }
+            }
+        }
+
+        stage('Clone service repo for fetching app details') {
+            steps {
+                script {
+                    log.info("Cloning App Repo - ${repo_name}, Branch - ${branch_name}")
+                    gitActions.clone("${WORKSPACE}", "${repo_name}", "${branch_name}")
+                    // sh 'ls -al'
+                    env.msg += '\n3. Service Repo Clone step was SUCCESSFUL.'
+                    // gitActions.branchCheckOut("${WORKSPACE}/${repo_name}", "${branch_name}")
+                }
+            }
+        }
+
+        stage('Collecting app details') {
+            steps {
+                script {
+                    dir("${repo_name}/deployments/") {
+                        sh 'ls -al'
+                        try {
+                            log.info("Reading BU, Team etc from ${repo_name}/deployments/${service}.yaml")
+                            bu = sh(returnStdout: true, script: "yq '.bu' ${service}.yaml").trim()
+                            team = sh(returnStdout: true, script: "yq '.team' ${service}.yaml").trim()
+                            priority = sh(returnStdout: true, script: "yq '.priority' ${service}.yaml").trim()
+                        }
+                        catch (Exception e) {
+                            env.msg += "\n\nFAILED -\n  Error while fetching app details from the app repo provided.\n  Full Error Details - ${e}"
+                            log.err("${env.msg}")
+                        }
+                    }
+                }
+            }
+        }
+
+        stage('Create alert-config.yaml') {
+            steps {
+                script {
+                    try {
+                        log.info('Creating alert-config.yaml')
+                        create_config()
+                        env.msg += '\n4. Create alert-config.yaml step was SUCCESSFUL.'
+                    }
+                    catch (Exception e) {
+                        env.msg += "\n\nFAILED -\n  Error while creating the alert-config.yaml.\n  Full Error Details - ${e}"
+                        log.err("${env.msg}")
+                    }
+                }
+            }
+        }
+
+        stage('Commit alert-config.yaml') {
+            steps {
+                script {
+                    new_branch_name = "${bu}-${team}-${service}"
+                    log.info("Pushing alert-config.yaml to Branch - ${new_branch_name}")
+                    // log.info("Pushing alert-config.yaml to Branch - ${service}")
+                    sh "rm -rf ${repo_name}"
+                    gitActions.branchCheckOut("${WORKSPACE}", "${new_branch_name}")
+                    // gitActions.branchCheckOut("${WORKSPACE}", "${service}")
+                    gitActions.add("${WORKSPACE}", '.')
+                    gitActions.codeCommit("${WORKSPACE}", "${new_branch_name}", "Updated ${alert_name} for ${new_branch_name}")
+                    gitActions.codePush("${WORKSPACE}", "${new_branch_name}")
+                    // gitActions.codeCommit("${WORKSPACE}", "${service}", 'Updated alert-config.yaml')
+                    // gitActions.codePush("${WORKSPACE}", "${service}")
+                    env.msg += '\n5. Pushing alert-config.yaml step is SUCCESSFUL. Please make sure to delete branch after PR Merge.'
+                }
+            }
+        }
+
+        stage('Raising PR') {
+            steps {
+                script {
+                    log.info("Creating PR -> Base - main; Head - ${new_branch_name}")
+                    pr_message = "Onboarding Alerts for ${service}"
+                    gitActions.createPR("${WORKSPACE}", 'terraform-infra', "${new_branch_name}", "${pr_message}")
+                    env.msg += "\n6. Creating PR, Head - ${new_branch_name} and Base - main is SUCCESSFUL."
+                }
+            }
+        }
+    }
+    post {
+        always {
+            cleanWs()
+        }
+    }
+}
+
+def create_config() {
+    dir("${WORKSPACE}/meesho/tools/vm-alerts-config/configmap") {
+        sh 'pwd'
+        sh "mkdir -p ${bu}/${team}/${service}/custom_alerts"
+        sh "touch ${bu}/${team}/${service}/custom_alerts/${service}-${alert_name}.yaml"
+
+        def config_template = libraryResource 'com/meesho/template-app-alert-configmap.yaml'
+        def config_binding = []
+
+        config_binding = [
+            'alert_name': "${env.alert_name}",
+            'query': "${query}",
+            'comparison_operator': "${comparison_operator}",
+            'threshold': "${threshold}",
+            'period': "${period}",
+            'severity': "${severity}",
+            'bu': "${bu}",
+            'team': "${team}",
+            'service': "${service}",
+            'environment': "${environment}",
+            'priority': "${priority}",
+            'summary': "${env.summary}"
+        ]
+        writeFile file:"${bu}/${team}/${service}/custom_alerts/${service}-${alert_name}.yaml", text: tokenize(config_template, config_binding)
+
+        sh "cat ${bu}/${team}/${service}/custom_alerts/${service}-${alert_name}.yaml"
+        sh 'ls -al;ls -al ..'
+    }
+}

helm-templates/victoria-metrics-alerts-config/Jenkinsfile_migration_sre_platform

@@ -0,0 +1,244 @@
+@Library('terraform-infra@main') _
+
+pipeline {
+    agent {
+        node {
+            label 'slave-01'
+        }
+    }
+
+    options {
+        timestamps()
+        ansiColor('xterm')
+    }
+
+    environment {
+        GITHUB_CRED = 'meesho-jenkins'
+    }
+
+    stages {
+        stage('Setup parameters') {
+            steps {
+                script {
+                    properties([
+                        parameters([
+                            string(
+                                defaultValue: '',
+                                name: 'bu',
+                                description: 'please enter the bu name',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'team',
+                                description: 'please enter the team name',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'priority',
+                                description: 'please enter the team name',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'approver1',
+                                description: 'please enter the approver name',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'approver2',
+                                description: 'please enter the second approver name',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'service',
+                                description: 'K8s Service/app name. Example: supplier-payment-web, ads-credit ',
+                                trim: true
+                            ),
+                            choice(
+                                choices: ['critical', 'warning'],
+                                description: 'NOTE: This is required',
+                                name: 'severity'
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'alert_name',
+                                description: 'Metric to be onbaorded to Alertmanager. Please use capital letters and underscore (_) only and don’t use spaces ( ). Example: HTTP_4XX_Alert_Per_API',
+                                trim: true
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'query',
+                                description: 'Copy the PromQL Query from existing Grafana Dashboard and paste it here',
+                                trim: true
+                            ),
+                            choice(
+                                choices: ['>=', '>', '=', '<=', '<'],
+                                description: 'NOTE: This is required',
+                                name: 'comparison_operator'
+                            ),
+                            choice(
+                                choices: ['prd', 'ftr', 'dev', 'int'],
+                                description: 'NOTE: This is required',
+                                name: 'environment'
+                            ),
+                            string(
+                                defaultValue: '',
+                                name: 'threshold',
+                                description: 'Threshold value of alerting. Example: 100, 200, 0.75 etc',
+                                trim: true
+                            ),
+                            choice(
+                                choices: ['1m', '2m', '5m', '10m'],
+                                description: 'NOTE: This is required. Default - 1m',
+                                name: 'period'
+                            )
+                        ])
+                    ])
+                }
+            }
+        }
+
+        stage('validation') {
+            steps {
+                script {
+                    env.msg = 'Started service alert onboarding'
+                    alert_name = alert_name.replaceAll("${service}", '')
+                    alert_name = alert_name.replaceAll(' ', '_')
+                    alert_name = alert_name.replaceAll('-', '_')
+                    alert_name = alert_name.replaceAll('__', '_')
+                    def metric_details = [
+                            'alert_name': "${alert_name}_EKS",
+                            'summary': "EKS ${alert_name} on {{ \$labels.service }}"
+                    ]
+                    env.alert_name = metric_details['alert_name']
+                    env.summary = metric_details['summary']
+                    env.int_env = 'main'
+
+                    log.info("""
+                        App BU Name - ${bu}
+                        App team Name - ${team}
+                        Approver1 Name - ${approver1}
+                        Approver2 Name - ${approver2}
+                        Threshold - ${threshold}
+                        Service Name - ${service}
+                        Severity - ${severity}
+                        Comparison Operator - ${comparison_operator}
+                        Env - ${environment}
+                        Period - ${period}
+                        Alert Name - ${env.alert_name}
+                        Query - ${query}
+                        Summary - ${env.summary}
+                    """)
+                    if (params.bu.isEmpty() || params.team.isEmpty() || params.threshold.isEmpty() || params.service.isEmpty()) {
+                        env.msg += '\n\nFAILED -\n  One or more input paramters are EMPTY/NULL.'
+                        log.err("${env.msg}")
+                    }
+                }
+            }
+        }
+
+        stage('Checkout Terraform-Infra repo') {
+            steps {
+                script {
+                    log.info('Checkout Terraform-Infra repo')
+                    gitActions.checkout()
+                    env.msg += '\n2. Terraform-Infra Repo checkout step was SUCCESSFUL.'
+                    // gitActions.branchCheckOut("${WORKSPACE}/${repo_name}", "${branch_name}")
+                }
+            }
+        }
+
+
+        stage('Create alert-config.yaml') {
+            steps {
+                script {
+                    try {
+                        log.info('Creating alert-config.yaml')
+                        create_config()
+                        env.msg += '\n4. Create alert-config.yaml step was SUCCESSFUL.'
+                    }
+                    catch (Exception e) {
+                        env.msg += "\n\nFAILED -\n  Error while creating the alert-config.yaml.\n  Full Error Details - ${e}"
+                        log.err("${env.msg}")
+                    }
+                }
+            }
+        }
+
+        stage('Commit alert-config.yaml') {
+            steps {
+                script {
+                    new_branch_name = "${bu}-${team}-${service}"
+                    log.info("Pushing alert-config.yaml to Branch - ${new_branch_name}")
+                    // log.info("Pushing alert-config.yaml to Branch - ${service}")
+                    // sh "rm -rf ${repo_name}"
+                    gitActions.branchCheckOut("${WORKSPACE}", "${new_branch_name}")
+                    // gitActions.branchCheckOut("${WORKSPACE}", "${service}")
+                    gitActions.add("${WORKSPACE}", '.')
+                    gitActions.codeCommit("${WORKSPACE}", "${new_branch_name}", "Updated ${alert_name} for ${new_branch_name}")
+                    gitActions.codePush("${WORKSPACE}", "${new_branch_name}")
+                    // gitActions.codeCommit("${WORKSPACE}", "${service}", 'Updated alert-config.yaml')
+                    // gitActions.codePush("${WORKSPACE}", "${service}")
+                    env.msg += '\n5. Pushing alert-config.yaml step is SUCCESSFUL. Please make sure to delete branch after PR Merge.'
+                }
+            }
+        }
+
+        stage('Raising PR') {
+            steps {
+                script {
+                    log.info("Creating PR -> Base - main; Head - ${new_branch_name}")
+                    pr_message = "Onboarding Alerts for ${service}"
+                    gitActions.createPR("${WORKSPACE}", 'terraform-infra', "${new_branch_name}", "${pr_message}")
+                    env.msg += "\n6. Creating PR, Head - ${new_branch_name} and Base - main is SUCCESSFUL."
+                }
+            }
+        }
+    }
+    post {
+        always {
+            cleanWs()
+        }
+        success {
+            def buildNumber = env.BUILD_NUMBER
+            def jobName = env.JOB_NAME
+            def curlCommand = "curl -X POST -d '{"buildnumber": "$buildNumber", "jobName": "$jobName"}' http://10.21.148.142:8002/AddReviewersToPR"
+            def response = sh(returnStdout: true, script: curlCommand)
+            echo "Response: ${response}"
+        }
+    }
+}
+
+def create_config() {
+    dir("${WORKSPACE}/meesho/tools/vm-alerts-config/configmap") {
+        sh 'pwd'
+        sh "mkdir -p ${bu}/${team}/${service}/custom_alerts"
+        sh "touch ${bu}/${team}/${service}/custom_alerts/${service}-${alert_name}.yaml"
+
+        def config_template = libraryResource 'com/meesho/template-app-alert-configmap.yaml'
+        def config_binding = []
+
+        config_binding = [
+            'alert_name': "${env.alert_name}",
+            'query': "${query}",
+            'comparison_operator': "${comparison_operator}",
+            'threshold': "${threshold}",
+            'period': "${period}",
+            'severity': "${severity}",
+            'bu': "${bu}",
+            'team': "${team}",
+            'service': "${service}",
+            'environment': "${environment}",
+            'priority': "${priority}",
+            'summary': "${env.summary}"
+        ]
+        writeFile file:"${bu}/${team}/${service}/custom_alerts/${service}-${alert_name}.yaml", text: tokenize(config_template, config_binding)
+
+        sh "cat ${bu}/${team}/${service}/custom_alerts/${service}-${alert_name}.yaml"
+        sh 'ls -al;ls -al ..'
+    }
+}

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy_prd-edge-proxy-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 4000
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-edge-proxy_prd-edge-proxy-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 800
+    for: 5m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-admin-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-admin_prd-edge-proxy-admin-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 2
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-admin
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-admin-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-admin_prd-edge-proxy-admin-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 5
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-admin
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-admin-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-edge-proxy-admin_prd-edge-proxy-admin-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 15000
+    for: 3m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-admin
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-int-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-int_prd-edge-proxy-int-primary_80",envoy_response_code=~"[4].*",envoy_response_code!~"429"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 900
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-int
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-int-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-int_prd-edge-proxy-int-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 200
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-int
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-int-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-edge-proxy-int_prd-edge-proxy-int-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 400
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-int
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-int-sale-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-int-sale_prd-edge-proxy-int-sale-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 250
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-int-sale
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-int-sale-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-int-sale_prd-edge-proxy-int-sale-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-int-sale
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-int-sale-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-edge-proxy-int-sale_prd-edge-proxy-int-sale-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 1000
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-int-sale
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-meesholink-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-meesholink_prd-edge-proxy-meesholink-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 15
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-meesholink
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-meesholink-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-edge-proxy-meesholink_prd-edge-proxy-meesholink-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 2500
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-meesholink
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-meesholink/custom_alerts/edge-proxy-meesholink-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-meesholink_prd-edge-proxy-meesholink-primary_80",envoy_response_code=~"[4].*",envoy_response_code!~"429"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-meesholink
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-processing-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-processing_prd-edge-proxy-processing-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 40
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-processing
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-processing-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-processing_prd-edge-proxy-processing-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-processing
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-processing-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-edge-proxy-processing_prd-edge-proxy-processing-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 9000
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-processing
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-sale-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-sale_prd-edge-proxy-sale-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 250
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-sale
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-sale-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-sale_prd-edge-proxy-sale-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-sale
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-sale-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-edge-proxy-sale_prd-edge-proxy-sale-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 1000
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-sale
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-secondary-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-secondary_prd-edge-proxy-secondary-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 300
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-secondary
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-secondary-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-edge-proxy-secondary_prd-edge-proxy-secondary-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 850
+    for: 4m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-secondary
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-secondary/custom_alerts/edge-proxy-secondary-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy-secondary_prd-edge-proxy-secondary-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 12000
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-secondary
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy-secondary/custom_alerts/edge-proxy-secondary-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-edge-proxy-secondary_prd-edge-proxy-secondary-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 1000
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy-secondary
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/central/shared/edge-proxy/custom_alerts/edge-proxy-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-edge-proxy_prd-edge-proxy-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 18000
+    for: 1m
+    labels:
+      severity: critical
+      bu: central
+      team: shared
+      service: edge-proxy
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-intelligence/eks-airflow.yaml

@@ -0,0 +1,128 @@
+groups:
+- name: EKS-Airflow KubernetesNodeReady
+  rules:
+  - alert: KubernetesNodeReady
+    expr: kube_node_status_condition{cluster=~"p-dataplatform-cluster",condition="Ready",status="true"} == 0
+    for: 3m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-intelligence
+      service: airflow-new
+      env: prd
+      priority: p0
+    annotations:
+      summary: Kubernetes Node ready (instance {{ $labels.instance }})
+      description: "Node {{ $labels.node }} has been unready for a long time\n LABELS = {{ $labels }}"
+
+  - alert: EKS-Airflow KubernetesMemoryPressure
+    expr: kube_node_status_condition{cluster=~"p-dataplatform-cluster",condition="MemoryPressure",status="true"} == 1
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-intelligence
+      env: prd
+      priority: p0
+      service: airflow-new
+    annotations:
+      summary: Kubernetes memory pressure (instance {{ $labels.instance }})
+      description: "{{ $labels.node }} has MemoryPressure condition\n  LABELS = {{ $labels }}"
+
+  - alert: EKS-Airflow KubernetesDiskPressure
+    expr: kube_node_status_condition{cluster=~"p-dataplatform-cluster",condition="DiskPressure",status="true"} == 1
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-intelligence
+      env: prd
+      priority: p0
+      service: airflow-new
+    annotations:
+      summary: Kubernetes disk pressure (instance {{ $labels.instance }})
+      description: "{{ $labels.node }} has DiskPressure condition\n  LABELS = {{ $labels }}"
+
+  - alert: Pod-CPU-Crossed-80%
+    expr: 100 * sum by (pod) (rate(container_cpu_usage_seconds_total{image!="",pod!="", namespace="prd-di-airflow-new"}[1m]))  / sum by (pod) (kube_pod_container_resource_limits{resource="cpu", job=~"kube-state-metrics-.*",namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd'} > 80
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-intelligence
+      env: prd
+      priority: p0
+      service: airflow-new
+    annotations:
+      summary: High CPU Utilisation for EKS Pod - {{ $labels.pod }}
+      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising CPU over {{ $value }}% in the last 1 min.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: Pod-Memory-Crossed-80%
+    expr: 100 * sum by (pod) (container_memory_working_set_bytes{image!="",namespace="prd-di-airflow-new"}) / sum by (pod) (kube_pod_container_resource_limits{resource="memory", job=~"kube-state-metrics-.*",namespace="prd-di-airflow-new"} > 0) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!="opencost", label_env=~'prod|prd'} > 80
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-intelligence
+      env: prd
+      priority: p0
+      service: airflow-new
+    annotations:
+      summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
+      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+  - alert: Pod-in-PodInitializing-State
+    expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='PodInitializing',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-intelligence
+      env: prd
+      priority: p0
+      service: airflow-new
+    annotations:
+      summary: EKS Pod - {{ $labels.pod }} is in PodInitializing State
+      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in PodInitializing state for last 2 mins.\n  STATE = PodInitializing\n  LABELS = {{ $labels }}"
+
+  - alert: Pod-in-CrashLoopBackOff-State
+    expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='CrashLoopBackOff',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-intelligence
+      env: prd
+      priority: p0
+      service: airflow-new
+    annotations:
+      summary: EKS Pod - {{ $labels.pod }} is in CrashLoopBackOff State
+      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff state for last 2 mins.\n  STATE = CrashLoopBackOff\n  LABELS = {{ $labels }}"
+
+  - alert: Pod-in-ImagePullBackOff-State
+    expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='ImagePullBackOff',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-intelligence
+      env: prd
+      priority: p0
+      service: airflow-new
+    annotations:
+      summary: EKS Pod - {{ $labels.pod }} is in ImagePullBackOff State
+      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in ImagePullBackOff state for last 2 mins.\n  STATE = ImagePullBackOff\n  LABELS = {{ $labels }}"
+
+  - alert: Pod-in-CreateContainerConfigError-State
+    expr: sum by (pod) (kube_pod_container_status_waiting_reason{reason='CreateContainerConfigError',namespace="prd-di-airflow-new"}) * on(pod) group_left(label_bu, label_team, label_service, label_env, label_priority, namespace, cluster) kube_pod_labels{kubernetes_namespace!='opencost', label_env=~'prod|prd'} == 1
+    for: 2m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-intelligence
+      env: prd
+      priority: p0
+      service: airflow-new
+    annotations:
+      summary: EKS Pod - {{ $labels.pod }} is in CreateContainerConfigError State
+      description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in CreateContainerConfigError state for last 2 mins.\n  STATE = CreateContainerConfigError\n  LABELS = {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-read/custom_alerts/ab-service-read-AB_HBASE_READ_FAILURE.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: AB_HBASE_READ_FAILURE_EKS
+    expr: increase(sum(HBASE_READ_FAILURE_value) by (table,operation) [1m]) >= 1
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-read
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS AB_HBASE_READ_FAILURE on {{ $labels.service }}"
+      description: "EKS AB_HBASE_READ_FAILURE on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-read/custom_alerts/ab-service-read-AB_HBASE_READ_FAILURE_FOR_SCAN_USER_AUDIENCE_MAP.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: AB_HBASE_READ_FAILURE_FOR_SCAN_USER_AUDIENCE_MAP_EKS
+    expr: increase(sum(HBASE_READ_FAILURE_value{table="user_audience_map",operation="scan"}) by (table,operation) [1h]) >= 1
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-read
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS AB_HBASE_READ_FAILURE_FOR_SCAN_USER_AUDIENCE_MAP on {{ $labels.service }}"
+      description: "EKS AB_HBASE_READ_FAILURE_FOR_SCAN_USER_AUDIENCE_MAP on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-read/custom_alerts/ab-service-read-AB_KAFKA_PRODUCER_AVG_LATENCY.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: AB_KAFKA_PRODUCER_AVG_LATENCY_EKS
+    expr: avg(kafka_producer_producer_metrics_request_latency_avg{app=~"prd-ab-service-read|prd-ab-service-write|prd-ab-worker-create|prd-ab-worker-abacus"})[1m] >= 100
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-read
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS AB_KAFKA_PRODUCER_AVG_LATENCY on {{ $labels.service }}"
+      description: "EKS AB_KAFKA_PRODUCER_AVG_LATENCY on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-read/custom_alerts/ab-service-read-AUDIENCE_ENTITY_MAP_MQ_SEND_EVENT_FAILURES.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: AUDIENCE_ENTITY_MAP_MQ_SEND_EVENT_FAILURES_EKS
+    expr: sum(increase(MQ_PRODUCER_SEND_EVENT_FAILURE_value{topic="entity.audience.map.update.topic"})[1m]) > 0
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-read
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS AUDIENCE_ENTITY_MAP_MQ_SEND_EVENT_FAILURES on {{ $labels.service }}"
+      description: "EKS AUDIENCE_ENTITY_MAP_MQ_SEND_EVENT_FAILURES on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-read/custom_alerts/ab-service-read-AUDIENCE_REFRESH_REDIS_QUEUE_LAG.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: AUDIENCE_REFRESH_REDIS_QUEUE_LAG_EKS
+    expr: sum(REDIS_QUEUE_SIZE_value) by (queue) [1h] >= 200
+    for: 10m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-read
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS AUDIENCE_REFRESH_REDIS_QUEUE_LAG on {{ $labels.service }}"
+      description: "EKS AUDIENCE_REFRESH_REDIS_QUEUE_LAG on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-read/custom_alerts/ab-service-read-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-ab-service-read_prd-ab-service-read-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50000
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-read
+      env: prd
+      priority: p1
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-read/custom_alerts/ab-service-read-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-ab-service-read_prd-ab-service-read-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 600
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-read
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-read/custom_alerts/ab-service-read-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-ab-service-read_prd-ab-service-read-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 40
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-read
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-read/custom_alerts/ab-service-read-POD_Memory_Crossed_EKS.yaml

@@ -0,0 +1,16 @@
+groups:
+  - name: EKS-App-Alerts
+    rules:
+      - alert: EKS-ab-service-read-POD_Memory_Crossed_Threshold-Alerts
+        expr: sum(container_memory_working_set_bytes{pod=~"prd-ab-service-read-.*",image!=""}) by (pod) / sum(kube_pod_container_resource_limits{pod=~"prd-ab-service-read-.*", resource=~'memory', job=~'kube-state-metrics-p-.*'}) by (pod) * 100 > 85
+        for: 1m
+        labels:
+          severity: critical
+          bu: dataengg
+          team: data-platform-experimentation
+          service: ab-service-read
+          env: prd
+          priority: p0
+        annotations:
+          summary: High Memory Utilisation for EKS Pod - {{ $labels.pod }}
+          description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been utilising Memory over {{ $value }}% in the last 1 min.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-write/custom_alerts/ab-service-write-AB_HBASE_CALL_QUEUE_TOO_BIG.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: AB_HBASE_CALL_QUEUE_TOO_BIG_EKS
+    expr: sum(increase(HBase_RegionServer_IPC_exceptions_callQueueTooBig{cluster="bac-p-ab-hbase"})[1m]) by (cluster) >= 3000
+    for: 2m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-write
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS AB_HBASE_CALL_QUEUE_TOO_BIG on {{ $labels.service }}"
+      description: "EKS AB_HBASE_CALL_QUEUE_TOO_BIG on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-write/custom_alerts/ab-service-write-AB_HBASE_WRITE_FAILURE.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: AB_HBASE_WRITE_FAILURE_EKS
+    expr: increase(sum(HBASE_WRITE_FAILURE_value) by (table,operation)[1m]) >= 1
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-write
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS AB_HBASE_WRITE_FAILURE on {{ $labels.service }}"
+      description: "EKS AB_HBASE_WRITE_FAILURE on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-write/custom_alerts/ab-service-write-AB_HBASE_WRITE_FAILURE_FOR_DELETE_USER_AUDIENCE_MAP.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: AB_HBASE_WRITE_FAILURE_FOR_DELETE_USER_AUDIENCE_MAP_EKS
+    expr: increase(sum(HBASE_WRITE_FAILURE_value{operation="deleteBulk",table="user_audience_map"}) by (table,operation)[1h]) >= 1
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-write
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS AB_HBASE_WRITE_FAILURE_FOR_DELETE_USER_AUDIENCE_MAP on {{ $labels.service }}"
+      description: "EKS AB_HBASE_WRITE_FAILURE_FOR_DELETE_USER_AUDIENCE_MAP on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-write/custom_alerts/ab-service-write-AB_HBASE_WRITE_FAILURE_FOR_UPDATE_USER_AUDIENCE_MAP.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: AB_HBASE_WRITE_FAILURE_FOR_UPDATE_USER_AUDIENCE_MAP_EKS
+    expr: increase(sum(HBASE_WRITE_FAILURE_value{operation="updateBulk",table="user_audience_map"}) by (table,operation)[1h]) >= 1
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-write
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS AB_HBASE_WRITE_FAILURE_FOR_UPDATE_USER_AUDIENCE_MAP on {{ $labels.service }}"
+      description: "EKS AB_HBASE_WRITE_FAILURE_FOR_UPDATE_USER_AUDIENCE_MAP on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-write/custom_alerts/ab-service-write-HTTP_4XX_Alert_Contour_EKS.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-ab-service-write_prd-ab-service-write-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-write
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour_EKS on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour_EKS on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-write/custom_alerts/ab-service-write-HTTP_5XX_Alert_Contour_EKS.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-ab-service-write_prd-ab-service-write-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 50
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform-experimentation
+      service: ab-service-write
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour_EKS on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour_EKS on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-write/custom_alerts/ab-service-write-P99_Dynamic_Audience_Add_Latency_Alert_EKS.yaml

@@ -0,0 +1,16 @@
+groups:
+  - name: EKS-App-Alerts
+    rules:
+      - alert: P99_Latency_Dynamic_Audience_Add_Alert_EKS
+        expr: histogram_quantile(0.99, sum(rate(CONTROLLER_execTime_bucket{uri=~"(/v2/entity/dynamic-audience/add)",service="ab-service-write"}[1m])) by (le, service, uri)) >= 500
+        for: 3m
+        labels:
+          severity: critical
+          bu: dataengg
+          team: data-platform-experimentation
+          service: ab-service-write
+          env: prd
+          priority: p0
+        annotations:
+          summary: "EKS P99_Dynamic_Audience_Add_Latency_Alert on {{ $labels.service }}"
+          description: "EKS P99_Dynamic_Audience_Add_Latency_Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform-experimentation/ab-service-write/custom_alerts/ab-service-write-P99_Update_Audience_Bulk_Latency_Alert_EKS.yaml

@@ -0,0 +1,16 @@
+groups:
+  - name: EKS-App-Alerts
+    rules:
+      - alert: P99_Latency_Dynami_Audience_Add_Alert_EKS
+        expr: histogram_quantile(0.99, sum(rate(CONTROLLER_execTime_bucket{uri=~"(/v2/entity/audience/update/bulk)",service="ab-service-write"}[1m])) by (le, service, uri)) >= 1500
+        for: 3m
+        labels:
+          severity: critical
+          bu: dataengg
+          team: data-platform-experimentation
+          service: ab-service-write
+          env: prd
+          priority: p0
+        annotations:
+          summary: "EKS P99_Update_Audience_Bulk_Latency_Alert on {{ $labels.service }}"
+          description: "EKS P99_Update_Audience_Bulk_Latency_Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/inhouse-ingestion-HTTP_4XX_Alert_Contour.yaml.bak

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-inhouse-ingestion_prd-inhouse-ingestion-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 110000
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: inhouse-ingestion
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/inhouse-ingestion-HTTP_5XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_EKS
+    expr: sum(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[5].*", service=~"prd-inhouse-ingestion", job=~"kubernetes-service-endpoints"}) by (cluster, service, ingress, job) >= 500
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: inhouse-ingestion
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 5XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 5XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/inhouse-ingestion-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-inhouse-ingestion_prd-inhouse-ingestion-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: inhouse-ingestion
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/inhouse-ingestion-Latency_P99_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_P99_Alert_EKS
+    expr: histogram_quantile(0.99, sum(nginx_ingress_controller_request_duration_seconds_bucket{cluster!="prod-ops-cluster",service=~"prd-inhouse-ingestion", job=~"kubernetes-service-endpoints"}) by (cluster, service, le, ingress, namespace, job)) >= 0.75
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: inhouse-ingestion
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}"
+      description: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/inhouse-ingestion-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-inhouse-ingestion_prd-inhouse-ingestion-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 450
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: inhouse-ingestion
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/messaging-api-external-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-messaging-api-external_prd-messaging-api-external-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: messaging-api-external
+      env: prd
+      priority: p1
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/messaging-api-external-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-messaging-api-external_prd-messaging-api-external-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: messaging-api-external
+      env: prd
+      priority: p1
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/messaging-api-external-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-messaging-api-external_prd-messaging-api-external-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 750
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: messaging-api-external
+      env: prd
+      priority: p1
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/messaging-api-internal-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-messaging-api-internal_prd-messaging-api-internal-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 450
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: messaging-api-internal
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/messaging-api-internal-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-messaging-api-internal_prd-messaging-api-internal-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: messaging-api-internal
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/messaging-api-internal-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-messaging-api-internal_prd-messaging-api-internal-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 125
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: messaging-api-internal
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/metastore-HTTP_4XX_Alert.yaml.bak

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[4].*", service=~"prd-metastore", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: metastore
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 4XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 4XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/metastore-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-metastore_prd-metastore-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: metastore
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/metastore-HTTP_5XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[5].*", service=~"prd-metastore", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: metastore
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 5XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 5XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/metastore-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-metastore_prd-metastore-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: metastore
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/metastore-Latency_P99_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_P99_Alert_EKS
+    expr: histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster!="prod-ops-cluster", service=~"prd-metastore", job=~"kubernetes-pods"}[1m])) by (cluster, service, le, ingress, namespace, job)) >= 0.75
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: metastore
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}"
+      description: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/metastore-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-metastore_prd-metastore-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 0.75
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: metastore
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/model-metastore-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-model-metastore_prd-model-metastore-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: model-metastore
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/model-metastore-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-model-metastore_prd-model-metastore-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: model-metastore
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/dataengg/data-platform/model-metastore-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,17 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-model-metastore_prd-model-metastore-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 750
+    for: 1m
+    labels:
+      severity: critical
+      bu: dataengg
+      team: data-platform
+      service: model-metastore
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}
+  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/atc-event-store-consumer-HTTP_4XX_Alert.yaml.bak

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[4].*", service=~"prd-atc-event-store-consumer", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: atc-event-store-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 4XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 4XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/atc-event-store-consumer-HTTP_5XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[5].*", service=~"prd-atc-event-store-consumer", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: atc-event-store-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 5XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 5XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/atc-event-store-consumer-Latency_P99_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_P99_Alert_EKS
+    expr: histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster!="prod-ops-cluster", service=~"prd-atc-event-store-consumer", job=~"kubernetes-pods"}[1m])) by (cluster, service, le, ingress, namespace, job)) >= 0.75
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: atc-event-store-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}"
+      description: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/click-event-store-consumer-HTTP_4XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[4].*", service=~"prd-click-event-store-consumer", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: click-event-store-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 4XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 4XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/click-event-store-consumer-HTTP_5XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[5].*", service=~"prd-click-event-store-consumer", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: click-event-store-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 5XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 5XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/click-event-store-consumer-Latency_P99_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_P99_Alert_EKS
+    expr: histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster!="prod-ops-cluster", service=~"prd-click-event-store-consumer", job=~"kubernetes-pods"}[1m])) by (cluster, service, le, ingress, namespace, job)) >= 0.75
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: click-event-store-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}"
+      description: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/dropoutnet-cg-api-HTTP_4XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[4].*", service=~"prd-dropoutnet-cg-api", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: dropoutnet-cg-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 4XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 4XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/dropoutnet-cg-api-HTTP_4XX_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Latency_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-dropoutnet-cg-api_prd-dropoutnet-cg-api-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: dropoutnet-cg-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/dropoutnet-cg-api-HTTP_5XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[5].*", service=~"prd-dropoutnet-cg-api", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: dropoutnet-cg-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 5XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 5XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/dropoutnet-cg-api-HTTP_5XX_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Latency_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-dropoutnet-cg-api_prd-dropoutnet-cg-api-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: dropoutnet-cg-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/dropoutnet-cg-api-Latency_P99_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_P99_Alert_EKS
+    expr: histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster!="prod-ops-cluster", service=~"prd-dropoutnet-cg-api", job=~"kubernetes-pods"}[1m])) by (cluster, service, le, ingress, namespace, job)) >= 0.50
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: dropoutnet-cg-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}"
+      description: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/dropoutnet-cg-api-P99_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: P99_Latency_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-dropoutnet-cg-api_prd-dropoutnet-cg-api-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 80
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: dropoutnet-cg-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS P99_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/dropoutnet-cg-consumer-HTTP_4XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[4].*", service=~"prd-dropoutnet-cg-consumer", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: dropoutnet-cg-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 4XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 4XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/dropoutnet-cg-consumer-HTTP_5XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[5].*", service=~"prd-dropoutnet-cg-consumer", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: dropoutnet-cg-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 5XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 5XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/dropoutnet-cg-consumer-Latency_P99_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_P99_Alert_EKS
+    expr: histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster!="prod-ops-cluster", service=~"prd-dropoutnet-cg-consumer", job=~"kubernetes-pods"}[1m])) by (cluster, service, le, ingress, namespace, job)) >= 0.75
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: dropoutnet-cg-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}"
+      description: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/entity-scoring-api/custom_alerts/entity-scoring-api-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-entity-scoring-api_prd-entity-scoring-api_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: entity-scoring-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/entity-scoring-api/custom_alerts/entity-scoring-api-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-entity-scoring-api_prd-entity-scoring-api_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: entity-scoring-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/entity-scoring-api/custom_alerts/entity-scoring-api-Latency_P99_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_P99_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(CONTROLLER_execTime_bucket{uri!~".*health.*",service="entity-scoring-api",instance=~"100.64.*"}[1m])) by (le, instance)) >= 750
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: entity-scoring-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}"
+      description: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/entity-scoring-consumer/custom_alerts/entity-scoring-consumer-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-entity-scoring-consumer_prd-entity-scoring-consumer_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: entity-scoring-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/entity-scoring-consumer/custom_alerts/entity-scoring-consumer-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-entity-scoring-consumer_prd-entity-scoring-consumer_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: entity-scoring-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/entity-scoring-consumer/custom_alerts/entity-scoring-consumer-Latency_P99_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_P99_Alert_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(CONTROLLER_execTime_bucket{uri!~".*health.*",service="entity-scoring-consumer",instance=~"100.64.*"}[1m])) by (le, instance)) >= 750
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: entity-scoring-consumer
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}"
+      description: "EKS Latency_P99_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/fy-ibcg-service-web/custom_alerts/Latency_P99_Alert_EKS_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_P99_Alert_EKS_Contour_EKS
+    expr: histogram_quantile(0.99, sum(rate(envoy_cluster_upstream_rq_time_bucket{envoy_cluster_name=~"prd-fy-ibcg-service_prd-fy-ibcg-service-primary_80"}[1m])) by (cluster, envoy_cluster_name, le, job)) >= 200
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: fy-ibcg-service-web
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency_P99_Alert_EKS_Contour on {{ $labels.service }}"
+      description: "EKS Latency_P99_Alert_EKS_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/fy-ibcg-service-web/custom_alerts/fy-ibcg-service-web-HTTP_4XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-fy-ibcg-service_prd-fy-ibcg-service-primary_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 20
+    for: 5m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: fy-ibcg-service-web
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/fy-ibcg-service-web/custom_alerts/fy-ibcg-service-web-HTTP_5XX_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-fy-ibcg-service_prd-fy-ibcg-service-primary_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 20
+    for: 5m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: fy-ibcg-service-web
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/item-scoring-api--topk-Latency_p99_Alert_Telegraf_EKS.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_p99_Alert_item-scoring_api_Telegraf_EKS
+    expr: histogram_quantile(0.99, sum(rate(CONTROLLER_execTime_bucket{service="item-scoring-api", uri="/api/v2/sorted-items/top-k", instance=~"100.64.*"}[1m])) by (le, uri)) >= 15
+    for: 5m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: item-scoring-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency_p99_Alert_item-scoring-api_Telegraf on {{ $labels.service }}"
+      description: "EKS Latency_p99_Alert_item-scoring-api_Telegraf on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/item-scoring-api-HTTP_4XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[4].*", service=~"prd-item-scoring-api", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: item-scoring-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 4XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 4XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/item-scoring-api-HTTP_4XX_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_4XX_Latency_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-item-scoring-api_prd-item-scoring-api_80",envoy_response_code=~"[4].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: item-scoring-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_4XX_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_4XX_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/item-scoring-api-HTTP_5XX_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Alert_EKS
+    expr: sum(rate(nginx_ingress_controller_requests{cluster!="prod-ops-cluster", status=~"[5].*", service=~"prd-item-scoring-api", job=~"kubernetes-pods"}[1m])) by (cluster, service, ingress, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: item-scoring-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP 5XX Alert on {{ $labels.service }}"
+      description: "EKS HTTP 5XX Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/item-scoring-api-HTTP_5XX_Latency_Alert_Contour.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_5XX_Latency_Alert_Contour_EKS
+    expr: sum(rate(envoy_cluster_upstream_rq{envoy_cluster_name=~"prd-item-scoring-api_prd-item-scoring-api_80",envoy_response_code=~"[5].*"}[1m])) by (cluster, envoy_cluster_name, job) * 60 >= 10
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: item-scoring-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_5XX_Latency_Alert_Contour on {{ $labels.service }}"
+      description: "EKS HTTP_5XX_Latency_Alert_Contour on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/item-scoring-api-HTTP_Non2xx_Alert_item-scoring-api_Telegraf_EKS.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: HTTP_Non2xx_Alert_item-scoring-api_Telegraf_EKS
+    expr: sum by (service,uri,httpCode) (CONTROLLER_execTime_count{service="item-scoring-api",instance=~"100.64.*",httpCode!~"2[0-9]{2}"} ) >= 50
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: item-scoring-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS HTTP_Non2xx_Alert_item-scoring-api_Telegraf on {{ $labels.service }}"
+      description: "EKS HTTP_Non2xx_Alert_item-scoring-api_Telegraf on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

helm-templates/victoria-metrics-alerts-config/configmap/datascience/ml-platform/item-scoring-api-Latency_P99_Alert.yaml

@@ -0,0 +1,16 @@
+groups:
+- name: EKS-App-Alerts
+  rules:
+  - alert: Latency_P99_Alert_EKS
+    expr: histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{cluster!="prod-ops-cluster", service=~"prd-item-scoring-api", job=~"kubernetes-pods"}[1m])) by (cluster, service, le, ingress, namespace, job)) >= 0.75
+    for: 1m
+    labels:
+      severity: critical
+      bu: datascience
+      team: ml-platform
+      service: item-scoring-api
+      env: prd
+      priority: p0
+    annotations:
+      summary: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}"
+      description: "EKS Latency/Response Time P99 Alert on {{ $labels.service }}  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"