From 3419cfba0cd534e53e29786d6ef22919ec14e9c0 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 26 Aug 2026 02:02:24 +0530 Subject: [PATCH] added files --- AGENTS.md | 156 ++ BUGS_AND_IMPROVEMENTS_REPORT.md | 188 ++ CLAUDE.md | 156 ++ README.md | 65 + ai-blitz/README.md | 45 + ai-blitz/_TEMPLATE.md | 20 + ai-blitz/task-01-securityScan-hardcoded-ip.md | 22 + ai-blitz/task-02-checkoutSubmodule-typo.md | 30 + .../task-03-onlyPushtoJfrog-jdk-versions.md | 22 + ...task-04-buildPipeline-agent-label-param.md | 26 + ...ask-05-cloudFunctionCICD-implementation.md | 40 + build.gradle | 38 + docs/SECURITY.md | 262 +++ docs/acronyms.md | 18 + ...-single-shared-library-for-all-services.md | 46 + ...ranch-name-as-sole-environment-selector.md | 58 + ...y-exceptions-in-separate-whitelist-repo.md | 60 + .../0004-fresh-whitelist-clone-per-build.md | 50 + ...config-only-change-detection-skip-build.md | 46 + ...ringmaster-mandatory-build-trigger-gate.md | 48 + ...-gitops-via-argocd-4-step-sync-sequence.md | 56 + ...ary-mandatory-for-tier1-services-in-prd.md | 56 + ...ap-auto-derived-from-pod-memory-request.md | 53 + ...ud-and-branch-namespaced-artifact-paths.md | 59 + ...er-identity-routes-post-build-callbacks.md | 66 + ...terpolated-helm-values-from-user-config.md | 61 + ...bles-gated-out-of-direct-jenkins-argocd.md | 62 + ...dependabot-critical-alerts-block-builds.md | 58 + ...-across-buildnode-groovy-and-dockerfile.md | 59 + ...6-per-environment-helm-chart-versioning.md | 60 + docs/adr/README.md | 43 + docs/architecture.md | 254 +++ docs/downstreams.md | 115 ++ docs/golden-prs.md | 108 ++ docs/index.md | 43 + docs/infrastructure.md | 67 + docs/review-learnings.md | 27 + docs/skills/README.md | 9 + docs/skills/build-failure-debugger/SPEC.md | 129 ++ docs/skills/library-impact-analyzer/SPEC.md | 73 + docs/skills/pipeline-tracer/SPEC.md | 112 ++ docs/tribal-knowledge.md | 133 ++ docs/wiki/.citation-index.json | 39 + docs/wiki/.drift-queue/.gitkeep | 0 docs/wiki/MANIFEST.md | 106 ++ docs/wiki/SCHEMA.md | 60 + docs/wiki/allowlist.yaml | 38 + docs/wiki/index.md | 56 + docs/wiki/log.md | 6 + docs/wiki/pages/01-ARCHITECTURE.md | 86 + docs/wiki/pages/01-overview.md | 57 + docs/wiki/pages/02-ENTRYPOINTS.md | 72 + docs/wiki/pages/02-entry-points.md | 40 + docs/wiki/pages/03-BUILD-STAGES.md | 78 + docs/wiki/pages/03-build-dispatch.md | 66 + docs/wiki/pages/04-DEPLOY-ARGOCD.md | 88 + docs/wiki/pages/04-deploy-flow.md | 87 + docs/wiki/pages/05-ENVIRONMENT-MAPPING.md | 70 + docs/wiki/pages/05-cross-cutting.md | 84 + docs/wiki/pages/06-CONFIG-POLICY.md | 82 + docs/wiki/pages/07-LANGUAGE-BUILDS.md | 87 + docs/wiki/pages/08-DOCKERFILE-TEMPLATES.md | 84 + docs/wiki/pages/09-INFRA-PODS.md | 81 + docs/wiki/pages/10-NOTIFICATIONS.md | 82 + docs/wiki/pages/adr/adr-index.md | 55 + docs/wiki/pages/build/build-dispatch.md | 53 + .../wiki/pages/build/config-only-detection.md | 46 + docs/wiki/pages/build/docker-tagging.md | 57 + docs/wiki/pages/build/node-paired-files.md | 49 + docs/wiki/pages/build/scm-variable-scope.md | 54 + docs/wiki/pages/concepts/observability.md | 55 + docs/wiki/pages/concepts/secrets-and-auth.md | 51 + docs/wiki/pages/concepts/whitelists.md | 31 + docs/wiki/pages/deploy/argocd-sync.md | 58 + .../pages/deploy/ringmaster-integration.md | 56 + docs/wiki/pages/infra/node-pool-selection.md | 54 + docs/wiki/pages/policy/cac-validation.md | 49 + docs/wiki/pages/policy/multi-tenancy.md | 50 + docs/wiki/pages/policy/whitelist-system.md | 50 + docs/wiki/pages/security/security-overview.md | 90 + gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 48462 bytes gradle/wrapper/gradle-wrapper.properties | 9 + gradlew | 248 +++ gradlew.bat | 82 + post-commit-scripts/._commit-metric.sh | Bin 0 -> 311 bytes post-commit-scripts/._runner.sh | Bin 0 -> 212 bytes post-commit-scripts/commit-metric.sh | 1551 +++++++++++++++++ post-commit-scripts/runner.sh | 47 + pre-commit-scripts/._cac-validate.sh | Bin 0 -> 212 bytes pre-commit-scripts/._runner.sh | Bin 0 -> 212 bytes pre-commit-scripts/._trufflehog-hook.sh | Bin 0 -> 212 bytes pre-commit-scripts/._yaakhook.sh | Bin 0 -> 212 bytes pre-commit-scripts/cac-validate.sh | 60 + pre-commit-scripts/runner.sh | 47 + pre-commit-scripts/trufflehog-hook.sh | 56 + pre-commit-scripts/yaakhook.sh | 30 + repository.yaml | 4 + resources/com/meesho/Dockerfile | 33 + resources/com/meesho/Jenkinsfile | 3 + resources/com/meesho/argoApp.yaml | 31 + resources/com/meesho/config.yaml | 14 + resources/com/meesho/cron-values.yaml | 99 ++ resources/com/meesho/deployJar.yaml | 203 +++ resources/com/meesho/deployment.yaml | 103 ++ resources/com/meesho/go-Dockerfile | 47 + resources/com/meesho/go-deployment.yaml | 65 + resources/com/meesho/go-values.yaml | 202 +++ resources/com/meesho/gradle-deployment.yaml | 103 ++ resources/com/meesho/java-Dockerfile | 33 + resources/com/meesho/node-Dockerfile | 58 + resources/com/meesho/node-deployment.yaml | 64 + resources/com/meesho/node-values.yaml | 184 ++ resources/com/meesho/php-Dockerfile | 35 + resources/com/meesho/php-deployment.yaml | 55 + resources/com/meesho/php-values.yaml | 169 ++ resources/com/meesho/python-2.7-Dockerfile | 29 + .../com/meesho/python-3.10.12-Dockerfile | 31 + resources/com/meesho/python-3.13-Dockerfile | 31 + resources/com/meesho/python-3.7-Dockerfile | 29 + resources/com/meesho/python-deployment.yaml | 64 + resources/com/meesho/python-values.yaml | 242 +++ resources/com/meesho/rust-Dockerfile | 58 + resources/com/meesho/rust-values.yaml | 206 +++ resources/com/meesho/validate_configs.py | 1233 +++++++++++++ resources/com/meesho/validate_configs_v2.py | 1261 ++++++++++++++ resources/com/meesho/values.yaml | 202 +++ resources/com/meesho/values_properties.yaml | 25 + resources/org/meesho/dev-pod.yaml | 47 + resources/org/meesho/prd-pod.yaml | 49 + resources/org/meesho/prd-sidecar-pod.yaml | 68 + resources/org/meesho/stg-pod.yaml | 48 + resources/org/meesho/stg-sidecar-pod.yaml | 67 + .../org/meesho/templates/maven-3.3-jdk-8.sh | 84 + resources/org/meesho/templates/node-12.22.sh | 50 + resources/org/meesho/toolchain-pod.yaml | 47 + review-learnings.md | 614 +++++++ review.md | 61 + sandbox-settings.xml | 0 schemas/asyncapi.yml | 0 src/com/meesho/stages/automationTest.groovy | 68 + src/com/meesho/stages/buildDocker.groovy | 68 + src/com/meesho/stages/buildGo.groovy | 412 +++++ src/com/meesho/stages/buildGradle.groovy | 542 ++++++ src/com/meesho/stages/buildMaven.groovy | 1059 +++++++++++ src/com/meesho/stages/buildNode.groovy | 568 ++++++ src/com/meesho/stages/buildObjHelper.groovy | 42 + src/com/meesho/stages/buildPhp.groovy | 104 ++ src/com/meesho/stages/buildPython.groovy | 221 +++ src/com/meesho/stages/buildRust.groovy | 315 ++++ src/com/meesho/stages/checkOut.groovy | 45 + src/com/meesho/stages/deployArgoCD.groovy | 906 ++++++++++ src/com/meesho/stages/deployJar.groovy | 32 + src/com/meesho/stages/deployRingmaster.groovy | 144 ++ src/com/meesho/stages/helmGenerator.groovy | 185 ++ src/com/meesho/stages/hotFix.groovy | 24 + .../meesho/stages/multiBranchPipeline.groovy | 73 + src/com/meesho/stages/notify.groovy | 206 +++ src/com/meesho/stages/securityScan.groovy | 24 + src/com/meesho/stages/validateBuTeam.groovy | 18 + src/com/meesho/utilities/addSSHKey.groovy | 28 + src/com/meesho/utilities/buTeamMapping.groovy | 125 ++ .../meesho/utilities/constructParam.groovy | 391 +++++ .../meesho/utilities/constructTemplate.groovy | 51 + .../meesho/utilities/dockerUtilities.groovy | 52 + .../meesho/utilities/getDockerParams.groovy | 61 + .../meesho/utilities/getYamlParameter.groovy | 16 + src/com/meesho/utilities/gitActions.groovy | 342 ++++ .../meesho/utilities/nodePoolSelection.groovy | 141 ++ .../com/meesho/stages/buildDocker.groovy | 2 + test/stubs/com/meesho/stages/buildGo.groovy | 2 + .../com/meesho/stages/buildGradle.groovy | 2 + .../stubs/com/meesho/stages/buildMaven.groovy | 2 + test/stubs/com/meesho/stages/buildNode.groovy | 2 + test/stubs/com/meesho/stages/buildPhp.groovy | 2 + .../com/meesho/stages/buildPython.groovy | 2 + test/stubs/com/meesho/stages/buildRust.groovy | 2 + .../com/meesho/utilities/addSSHKey.groovy | 5 + .../com/meesho/utilities/buTeamMapping.groovy | 5 + .../meesho/utilities/constructTemplate.groovy | 5 + .../meesho/utilities/dockerUtilities.groovy | 5 + .../meesho/utilities/getDockerParams.groovy | 5 + .../meesho/utilities/getYamlParameter.groovy | 5 + .../com/meesho/stages/BuildNodeTest.groovy | 32 + .../meesho/stages/BuildObjHelperTest.groovy | 165 ++ .../com/meesho/stages/BuildRustTest.groovy | 70 + .../meesho/utilities/BuTeamMappingTest.groovy | 207 +++ .../utilities/ConstructParamTest.groovy | 269 +++ vars/buildDockerGroovyGke.groovy | 164 ++ vars/buildPipeline.groovy | 43 + vars/cdHookRunner.groovy | 195 +++ vars/cloudFunctionCICD.groovy | 30 + vars/createEKSconfigs.groovy | 24 + vars/eksCICD.groovy | 150 ++ vars/gcpMigration.groovy | 278 +++ vars/gkeCICD.groovy | 202 +++ vars/log.groovy | 24 + vars/onlyPushtoJfrog.groovy | 115 ++ vars/runHooks.groovy | 160 ++ vars/stageName.groovy | 16 + vars/triggerCoverageOnly.groovy | 92 + 200 files changed, 22132 insertions(+) create mode 100644 AGENTS.md create mode 100644 BUGS_AND_IMPROVEMENTS_REPORT.md create mode 100644 CLAUDE.md create mode 100644 ai-blitz/README.md create mode 100644 ai-blitz/_TEMPLATE.md create mode 100644 ai-blitz/task-01-securityScan-hardcoded-ip.md create mode 100644 ai-blitz/task-02-checkoutSubmodule-typo.md create mode 100644 ai-blitz/task-03-onlyPushtoJfrog-jdk-versions.md create mode 100644 ai-blitz/task-04-buildPipeline-agent-label-param.md create mode 100644 ai-blitz/task-05-cloudFunctionCICD-implementation.md create mode 100644 build.gradle create mode 100644 docs/SECURITY.md create mode 100644 docs/acronyms.md create mode 100644 docs/adr/0001-single-shared-library-for-all-services.md create mode 100644 docs/adr/0002-branch-name-as-sole-environment-selector.md create mode 100644 docs/adr/0003-policy-exceptions-in-separate-whitelist-repo.md create mode 100644 docs/adr/0004-fresh-whitelist-clone-per-build.md create mode 100644 docs/adr/0005-config-only-change-detection-skip-build.md create mode 100644 docs/adr/0006-ringmaster-mandatory-build-trigger-gate.md create mode 100644 docs/adr/0007-gitops-via-argocd-4-step-sync-sequence.md create mode 100644 docs/adr/0008-canary-mandatory-for-tier1-services-in-prd.md create mode 100644 docs/adr/0009-jvm-heap-auto-derived-from-pod-memory-request.md create mode 100644 docs/adr/0010-cloud-and-branch-namespaced-artifact-paths.md create mode 100644 docs/adr/0011-build-user-identity-routes-post-build-callbacks.md create mode 100644 docs/adr/0012-string-interpolated-helm-values-from-user-config.md create mode 100644 docs/adr/0013-multi-zone-deployables-gated-out-of-direct-jenkins-argocd.md create mode 100644 docs/adr/0014-open-dependabot-critical-alerts-block-builds.md create mode 100644 docs/adr/0015-node-install-logic-paired-across-buildnode-groovy-and-dockerfile.md create mode 100644 docs/adr/0016-per-environment-helm-chart-versioning.md create mode 100644 docs/adr/README.md create mode 100644 docs/architecture.md create mode 100644 docs/downstreams.md create mode 100644 docs/golden-prs.md create mode 100644 docs/index.md create mode 100644 docs/infrastructure.md create mode 100644 docs/review-learnings.md create mode 100644 docs/skills/README.md create mode 100644 docs/skills/build-failure-debugger/SPEC.md create mode 100644 docs/skills/library-impact-analyzer/SPEC.md create mode 100644 docs/skills/pipeline-tracer/SPEC.md create mode 100644 docs/tribal-knowledge.md create mode 100644 docs/wiki/.citation-index.json create mode 100644 docs/wiki/.drift-queue/.gitkeep create mode 100644 docs/wiki/MANIFEST.md create mode 100644 docs/wiki/SCHEMA.md create mode 100644 docs/wiki/allowlist.yaml create mode 100644 docs/wiki/index.md create mode 100644 docs/wiki/log.md create mode 100644 docs/wiki/pages/01-ARCHITECTURE.md create mode 100644 docs/wiki/pages/01-overview.md create mode 100644 docs/wiki/pages/02-ENTRYPOINTS.md create mode 100644 docs/wiki/pages/02-entry-points.md create mode 100644 docs/wiki/pages/03-BUILD-STAGES.md create mode 100644 docs/wiki/pages/03-build-dispatch.md create mode 100644 docs/wiki/pages/04-DEPLOY-ARGOCD.md create mode 100644 docs/wiki/pages/04-deploy-flow.md create mode 100644 docs/wiki/pages/05-ENVIRONMENT-MAPPING.md create mode 100644 docs/wiki/pages/05-cross-cutting.md create mode 100644 docs/wiki/pages/06-CONFIG-POLICY.md create mode 100644 docs/wiki/pages/07-LANGUAGE-BUILDS.md create mode 100644 docs/wiki/pages/08-DOCKERFILE-TEMPLATES.md create mode 100644 docs/wiki/pages/09-INFRA-PODS.md create mode 100644 docs/wiki/pages/10-NOTIFICATIONS.md create mode 100644 docs/wiki/pages/adr/adr-index.md create mode 100644 docs/wiki/pages/build/build-dispatch.md create mode 100644 docs/wiki/pages/build/config-only-detection.md create mode 100644 docs/wiki/pages/build/docker-tagging.md create mode 100644 docs/wiki/pages/build/node-paired-files.md create mode 100644 docs/wiki/pages/build/scm-variable-scope.md create mode 100644 docs/wiki/pages/concepts/observability.md create mode 100644 docs/wiki/pages/concepts/secrets-and-auth.md create mode 100644 docs/wiki/pages/concepts/whitelists.md create mode 100644 docs/wiki/pages/deploy/argocd-sync.md create mode 100644 docs/wiki/pages/deploy/ringmaster-integration.md create mode 100644 docs/wiki/pages/infra/node-pool-selection.md create mode 100644 docs/wiki/pages/policy/cac-validation.md create mode 100644 docs/wiki/pages/policy/multi-tenancy.md create mode 100644 docs/wiki/pages/policy/whitelist-system.md create mode 100644 docs/wiki/pages/security/security-overview.md create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100644 gradlew create mode 100644 gradlew.bat create mode 100644 post-commit-scripts/._commit-metric.sh create mode 100644 post-commit-scripts/._runner.sh create mode 100644 post-commit-scripts/commit-metric.sh create mode 100644 post-commit-scripts/runner.sh create mode 100644 pre-commit-scripts/._cac-validate.sh create mode 100644 pre-commit-scripts/._runner.sh create mode 100644 pre-commit-scripts/._trufflehog-hook.sh create mode 100644 pre-commit-scripts/._yaakhook.sh create mode 100644 pre-commit-scripts/cac-validate.sh create mode 100644 pre-commit-scripts/runner.sh create mode 100644 pre-commit-scripts/trufflehog-hook.sh create mode 100644 pre-commit-scripts/yaakhook.sh create mode 100644 repository.yaml create mode 100644 resources/com/meesho/Dockerfile create mode 100644 resources/com/meesho/Jenkinsfile create mode 100644 resources/com/meesho/argoApp.yaml create mode 100644 resources/com/meesho/config.yaml create mode 100644 resources/com/meesho/cron-values.yaml create mode 100644 resources/com/meesho/deployJar.yaml create mode 100644 resources/com/meesho/deployment.yaml create mode 100644 resources/com/meesho/go-Dockerfile create mode 100644 resources/com/meesho/go-deployment.yaml create mode 100644 resources/com/meesho/go-values.yaml create mode 100644 resources/com/meesho/gradle-deployment.yaml create mode 100644 resources/com/meesho/java-Dockerfile create mode 100644 resources/com/meesho/node-Dockerfile create mode 100644 resources/com/meesho/node-deployment.yaml create mode 100644 resources/com/meesho/node-values.yaml create mode 100644 resources/com/meesho/php-Dockerfile create mode 100644 resources/com/meesho/php-deployment.yaml create mode 100644 resources/com/meesho/php-values.yaml create mode 100644 resources/com/meesho/python-2.7-Dockerfile create mode 100644 resources/com/meesho/python-3.10.12-Dockerfile create mode 100644 resources/com/meesho/python-3.13-Dockerfile create mode 100644 resources/com/meesho/python-3.7-Dockerfile create mode 100644 resources/com/meesho/python-deployment.yaml create mode 100644 resources/com/meesho/python-values.yaml create mode 100644 resources/com/meesho/rust-Dockerfile create mode 100644 resources/com/meesho/rust-values.yaml create mode 100644 resources/com/meesho/validate_configs.py create mode 100644 resources/com/meesho/validate_configs_v2.py create mode 100644 resources/com/meesho/values.yaml create mode 100644 resources/com/meesho/values_properties.yaml create mode 100644 resources/org/meesho/dev-pod.yaml create mode 100644 resources/org/meesho/prd-pod.yaml create mode 100644 resources/org/meesho/prd-sidecar-pod.yaml create mode 100644 resources/org/meesho/stg-pod.yaml create mode 100644 resources/org/meesho/stg-sidecar-pod.yaml create mode 100644 resources/org/meesho/templates/maven-3.3-jdk-8.sh create mode 100644 resources/org/meesho/templates/node-12.22.sh create mode 100644 resources/org/meesho/toolchain-pod.yaml create mode 100644 review-learnings.md create mode 100644 review.md create mode 100644 sandbox-settings.xml create mode 100644 schemas/asyncapi.yml create mode 100644 src/com/meesho/stages/automationTest.groovy create mode 100644 src/com/meesho/stages/buildDocker.groovy create mode 100644 src/com/meesho/stages/buildGo.groovy create mode 100644 src/com/meesho/stages/buildGradle.groovy create mode 100644 src/com/meesho/stages/buildMaven.groovy create mode 100644 src/com/meesho/stages/buildNode.groovy create mode 100644 src/com/meesho/stages/buildObjHelper.groovy create mode 100644 src/com/meesho/stages/buildPhp.groovy create mode 100644 src/com/meesho/stages/buildPython.groovy create mode 100644 src/com/meesho/stages/buildRust.groovy create mode 100644 src/com/meesho/stages/checkOut.groovy create mode 100644 src/com/meesho/stages/deployArgoCD.groovy create mode 100644 src/com/meesho/stages/deployJar.groovy create mode 100644 src/com/meesho/stages/deployRingmaster.groovy create mode 100644 src/com/meesho/stages/helmGenerator.groovy create mode 100644 src/com/meesho/stages/hotFix.groovy create mode 100644 src/com/meesho/stages/multiBranchPipeline.groovy create mode 100644 src/com/meesho/stages/notify.groovy create mode 100644 src/com/meesho/stages/securityScan.groovy create mode 100644 src/com/meesho/stages/validateBuTeam.groovy create mode 100644 src/com/meesho/utilities/addSSHKey.groovy create mode 100644 src/com/meesho/utilities/buTeamMapping.groovy create mode 100644 src/com/meesho/utilities/constructParam.groovy create mode 100644 src/com/meesho/utilities/constructTemplate.groovy create mode 100644 src/com/meesho/utilities/dockerUtilities.groovy create mode 100644 src/com/meesho/utilities/getDockerParams.groovy create mode 100644 src/com/meesho/utilities/getYamlParameter.groovy create mode 100644 src/com/meesho/utilities/gitActions.groovy create mode 100644 src/com/meesho/utilities/nodePoolSelection.groovy create mode 100644 test/stubs/com/meesho/stages/buildDocker.groovy create mode 100644 test/stubs/com/meesho/stages/buildGo.groovy create mode 100644 test/stubs/com/meesho/stages/buildGradle.groovy create mode 100644 test/stubs/com/meesho/stages/buildMaven.groovy create mode 100644 test/stubs/com/meesho/stages/buildNode.groovy create mode 100644 test/stubs/com/meesho/stages/buildPhp.groovy create mode 100644 test/stubs/com/meesho/stages/buildPython.groovy create mode 100644 test/stubs/com/meesho/stages/buildRust.groovy create mode 100644 test/stubs/com/meesho/utilities/addSSHKey.groovy create mode 100644 test/stubs/com/meesho/utilities/buTeamMapping.groovy create mode 100644 test/stubs/com/meesho/utilities/constructTemplate.groovy create mode 100644 test/stubs/com/meesho/utilities/dockerUtilities.groovy create mode 100644 test/stubs/com/meesho/utilities/getDockerParams.groovy create mode 100644 test/stubs/com/meesho/utilities/getYamlParameter.groovy create mode 100644 test/unit/com/meesho/stages/BuildNodeTest.groovy create mode 100644 test/unit/com/meesho/stages/BuildObjHelperTest.groovy create mode 100644 test/unit/com/meesho/stages/BuildRustTest.groovy create mode 100644 test/unit/com/meesho/utilities/BuTeamMappingTest.groovy create mode 100644 test/unit/com/meesho/utilities/ConstructParamTest.groovy create mode 100644 vars/buildDockerGroovyGke.groovy create mode 100644 vars/buildPipeline.groovy create mode 100644 vars/cdHookRunner.groovy create mode 100644 vars/cloudFunctionCICD.groovy create mode 100644 vars/createEKSconfigs.groovy create mode 100644 vars/eksCICD.groovy create mode 100644 vars/gcpMigration.groovy create mode 100644 vars/gkeCICD.groovy create mode 100644 vars/log.groovy create mode 100644 vars/onlyPushtoJfrog.groovy create mode 100644 vars/runHooks.groovy create mode 100644 vars/stageName.groovy create mode 100644 vars/triggerCoverageOnly.groovy diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ee81bb6 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,156 @@ + + +# devops-lib — Claude guide + +Jenkins shared library that backs every Meesho service's CI/CD pipeline. +Loaded by Jenkins as a Global Library; consumed by service repos through a +`@Library('devops-lib')` import in their `Jenkinsfile`. There is **no local +build, test, or run command** — the library only executes inside Jenkins. + +## What this repo is + +- Code is **Groovy** (Jenkins CPS-style, with `@NonCPS` islands). +- `vars/*.groovy` are Jenkins shared-library **globals** — the entry points + service repos call (e.g. `buildPipeline { ... }`, `eksCICD { ... }`). +- `src/com/meesho/stages/**.groovy` — stage implementations dispatched from + the globals (`buildMaven`, `buildGo`, `deployArgoCD`, `notify`, …). +- `src/com/meesho/utilities/**.groovy` — shared helpers (`constructParam`, + `gitActions`, `nodePoolSelection`, `constructTemplate`). +- `resources/com/meesho/**` — per-language `Dockerfile`, `*-deployment.yaml`, + `*-values.yaml` templates rendered into the service repo at deploy time. +- `resources/org/meesho/**-pod.yaml` — Jenkins build-agent pod templates + selected by `env.INFRA_ENV` (`dev` / `stg` / `prd`). + +## Architecture + +Long-form architecture and data flow lives in +[`docs/architecture.md`](docs/architecture.md). Read it before changing any +file under `src/com/meesho/stages/` or `vars/`. + +Pre-existing context worth reading: + +- [`docs/acronyms.md`](docs/acronyms.md) — repo-specific acronyms (BU, + BUILDKIT, CAC, GCPP, INFRA, …). +- [`docs/tribal-knowledge.md`](docs/tribal-knowledge.md) — load-bearing + non-obvious conventions (whitelist git clones, JVM memory auto-calc, canary + enforcement, `ringmaster-bot` user-id switch, etc.). +- [`review-learnings.md`](review-learnings.md) — PR-review-derived rules, + some graduated into the NEVER DO section below. +- [`BUGS_AND_IMPROVEMENTS_REPORT.md`](BUGS_AND_IMPROVEMENTS_REPORT.md) — + catalogue of known bugs / debt; do not silently "fix" these without + reading the linked PR history. + +## Quick reference + +| Task | Command | +|---|---| +| Sanity-check Groovy syntax | `groovy -e 'load "vars/buildPipeline.groovy"'` (not wired in CI; use only as a local lint) | +| Validate Helm value templates | runs inside Jenkins via `validate_configs.py` (no standalone CLI) | +| Run the library | Cannot run locally — push a branch, point a Jenkins job at `@Library('devops-lib@')`, trigger from a service repo | + +**No `Makefile`, no `build.gradle`, no `pom.xml`, no `Dockerfile`, no +`Jenkinsfile`, no `package.json`, no `.github/workflows/` at the repo root.** +A Jenkins job test-loads this library; there is no project-local equivalent. + +**No test suite.** `BUGS_AND_IMPROVEMENTS_REPORT.md` flags this as a known +high-priority gap. Do not fabricate `./gradlew test` or similar — they do not +exist. + +## Stack & tools + +- **Groovy** — Jenkins CPS engine. Beware: any `@NonCPS` method must not be + called across a serialisation boundary (e.g., inside a `parallel` closure) + without the closure itself being `@NonCPS`. +- **Jenkins shared-library layout** — `vars/.groovy` exposes `` + as a global step; `src/com/meesho/**.groovy` is classpath-loaded. +- **Helm / ArgoCD** — service deploys go through + `deployArgoCD.groovy` → `update_helm_repo` → `refresh_app_of_apps` → + `refresh_and_sync`. Step order is load-bearing + ([tribal-knowledge §10](docs/tribal-knowledge.md)). +- **Turbo-Turtle / Ringmaster** — `deployRingmaster.groovy` routes callbacks + based purely on `getCause(UserIdCause).getUserId() == "ringmaster-bot"`. + Renaming that user id silently breaks the routing. +- **Validation** — `resources/com/meesho/validate_configs.py` (1207 lines) is + a monolithic Python validator invoked from Groovy via `sh`. +- **Whitelists** — `getWhitelistedRepos()` does a fresh `git clone` of + `Meesho/whitelists` **on every call** (no caching). Five whitelist gates + each clone independently — this is intentional, do not refactor. + +## Critical conventions + +1. **`vars/` is the public API.** Adding a new entry point means adding a new + file under `vars/`. Renaming or removing a `vars/` global is a breaking + change for every service Jenkinsfile. +2. **`build_tool` switch lives in `src/com/meesho/stages/buildObjHelper.groovy`.** + Supported values today are `maven`, `gradle`, `docker`, `python-*`, + `node-*`, `go*`, `php`. There is no `sbt`, no `rust`. Unknown values fall + through to `defaultBuild` silently. +3. **`@NonCPS` rule.** `constructTemplate._construct()` is annotated + `@NonCPS` because it uses `groovy.text.SimpleTemplateEngine` (Java, + non-serialisable). Keep it `@NonCPS` and call it from a CPS-safe wrapper. +4. **JVM memory auto-calc.** `update_helm_repo` derives `xms == xmx == + memory_limit * 0.5`. Do not hard-code `-Xmx` in `JAVA_OPTS`. +5. **Canary is mandatory for `priority_v2: sp0` / `up0` in prd.** No + whitelist, no bypass. The build fails fast before the Helm update. +6. **PR vs branch detection.** Use `env.CHANGE_ID` (set by the GitHub Branch + Source plugin), never `env.BRANCH_NAME =~ /PR-/`. +7. **Hard-coded `'Meesho'` org.** Many helpers in `gitActions.groovy` / + `constructParam.groovy` embed the GitHub org name as a literal — keep this + stable. Do not parametrise without a coordinated rollout. +8. **Pod selection.** `env.INFRA_ENV` selects the Jenkins agent pod template + via `libraryResource("org/meesho/${env.INFRA_ENV}-pod.yaml")`. Don't add + inline `podTemplate` blocks — they bypass the central agent inventory. + +## NEVER DO + +These rules come from PR-review history (`review-learnings.md`) and +post-incident notes (`docs/tribal-knowledge.md`). Each links to its source. + +- **Never refactor `getWhitelistedRepos` to cache clones across calls** + without confirming the freshness guarantee is no longer required + ([tribal-knowledge §1](docs/tribal-knowledge.md)). +- **Never rename `"ringmaster-bot"`** in `deployRingmaster.groovy` without + coordinating with the Ringmaster team — it's a load-bearing string + ([tribal-knowledge §2](docs/tribal-knowledge.md)). +- **Never set `xms` / `xmx` manually in service Helm values** — the pipeline + computes them from `memory_limit`; manual values collide + ([tribal-knowledge §4](docs/tribal-knowledge.md)). +- **Never print or `cat` an SSH private key to stdout / logs** + (review-learnings `P0_NO_PRIVATE_KEY_LEAK`, PR #634). Always + `withCredentials { ... }` and write to a 0600 file. +- **Never hard-code bare IPs as curl/HTTP targets in pipeline code.** + (review-learnings `P0_HARDCODED_IP_IN_PIPELINE`, PR #681; known existing + exception: `securityScan.groovy:11` flagged for remediation.) Use + DNS-resolvable hostnames. +- **Never bypass canary on `sp0`/`up0` prd deploys** + ([tribal-knowledge §5](docs/tribal-knowledge.md)). +- **Never inline the Turbo-Turtle JSON payload in `curl` differently from + the current `-d '$newCICD_JSON'` pattern** without re-verifying bash + escaping ([tribal-knowledge §6](docs/tribal-knowledge.md)). +- **Never use `--no-verify` on `git commit`** — pre-commit hooks include + TruffleHog secret scanning and CAC validation. Bypassing is a P0 + policy violation. +- **Never commit to `main` or `develop` directly** — those branches are + protected; all changes go via PR. + +## Where to ask + +- `#ci-cd-status` Slack channel — pipeline failures, library questions. +- `@maintainer` Slack handle is required in every pipeline `param` map + (consumed by `src/com/meesho/stages/notify.groovy:9`). + +## Pointers + +- [docs/architecture.md](docs/architecture.md) — full module-by-module map. +- [docs/acronyms.md](docs/acronyms.md) — repo-specific abbreviations. +- [docs/tribal-knowledge.md](docs/tribal-knowledge.md) — non-obvious patterns. +- [review-learnings.md](review-learnings.md) — PR-review-derived rules, + graduation candidates for this file's NEVER DO section. +- [review.md](review.md) — review process / rubric. +- [BUGS_AND_IMPROVEMENTS_REPORT.md](BUGS_AND_IMPROVEMENTS_REPORT.md) — known + bug catalogue; check before "fixing" anything that looks suspect. diff --git a/BUGS_AND_IMPROVEMENTS_REPORT.md b/BUGS_AND_IMPROVEMENTS_REPORT.md new file mode 100644 index 0000000..0e69e6a --- /dev/null +++ b/BUGS_AND_IMPROVEMENTS_REPORT.md @@ -0,0 +1,188 @@ +# DevOps Library - Bugs and Improvements Report - Generated By Cursor AI + +## Executive Summary + +This report documents critical bugs, security vulnerabilities, and improvement opportunities found in the Meesho DevOps Jenkins shared library. The codebase shows signs of technical debt with multiple critical issues that need immediate attention. + +## Critical Bugs (High Priority) + +### 1. **Hard-coded Branch Name Bug** 🔴 +- **File**: `src/com/meesho/stages/buildGradle.groovy` +- **Line**: 252 +- **Issue**: `branch_name = 'repo'` - Hard-coded instead of using actual branch name +- **Impact**: JFrog deployment will always think it's not on master/main branch +- **Fix**: Change to `branch_name = "${env.BRANCH_NAME}"` + +### 2. **Method Name Typo** 🔴 +- **File**: `src/com/meesho/stages/checkOut.groovy` +- **Line**: 22 +- **Issue**: Method name `chekoutSubmodule` should be `checkoutSubmodule` +- **Impact**: Will cause runtime errors if this method is called +- **Fix**: Rename method to correct spelling + +### 3. **XML Query on Non-XML Files** 🔴 +- **File**: `src/com/meesho/stages/buildGradle.groovy` +- **Lines**: Multiple locations +- **Issue**: Using `xq` (XML query) command on `build.gradle` files +- **Impact**: Will fail as Gradle files are not XML format +- **Fix**: Parse Gradle files appropriately or use Gradle APIs + +### 4. **Exception Type Typo** 🔴 +- **File**: `src/com/meesho/stages/helmGenerator.groovy` +- **Line**: 102 +- **Issue**: `catch (Exceptione)` - Typo in Exception class name +- **Impact**: Syntax error, code won't compile +- **Fix**: Change to `catch (Exception e)` + +### 5. **Incomplete JDK Version Handling** 🟡 +- **File**: `vars/onlyPushtoJfrog.groovy` +- **Lines**: 48-53 +- **Issue**: Only handles JDK 8 and 17, but allows JDK 11 and 21 in parameters +- **Impact**: JDK 11 and 21 users will default to JDK 8 +- **Fix**: Add cases for all supported JDK versions + +### 6. **Duplicate Map Key** 🟡 +- **File**: `src/com/meesho/stages/buildPython.groovy` +- **Lines**: 76-79 +- **Issue**: `buildRegistry` key defined twice in same map +- **Impact**: First value will be overwritten +- **Fix**: Remove duplicate key + +## Security Vulnerabilities (Critical) + +### 1. **Hard-coded IP Address** 🔴 +- **File**: `src/com/meesho/stages/securityScan.groovy` +- **Line**: 12 +- **Issue**: Hard-coded IP `172.31.5.29:63232` +- **Impact**: Security risk, inflexible configuration +- **Fix**: Move to configuration/environment variable + +### 2. **Password Exposure in Process List** 🔴 +- **Files**: Multiple locations +- **Issue**: ECR and ArgoCD login commands expose passwords +- **Examples**: + ```groovy + sh "aws ecr get-login-password | docker login --password-stdin" + sh "argocd login --password ${ARGO_PASSWORD}" + ``` +- **Impact**: Passwords visible in process list and logs +- **Fix**: Use secure credential handling methods + +### 3. **SSH Key Security Issues** 🔴 +- **File**: `src/com/meesho/utilities/addSSHKey.groovy` +- **Issues**: + - SSH key written to file before permissions are set (race condition) + - No cleanup of SSH key file after use + - Key stored in plaintext +- **Fix**: Set permissions atomically, ensure cleanup, use agent forwarding + +### 4. **Unsafe File Deletion** 🟡 +- **Multiple files** +- **Issue**: `rm -rf *` commands without safeguards +- **Impact**: Could delete unintended files +- **Fix**: Use specific file paths, add safety checks + +## Code Quality Issues + +### 1. **Security Scan Disabled** 🔴 +- **File**: `vars/buildPipeline.groovy` +- **Line**: 23 +- **Issue**: Security scan is commented out +- **Impact**: No security validation in CI/CD pipeline +- **Fix**: Re-enable or remove with proper documentation + +### 2. **Incomplete Implementation** 🟡 +- **File**: `vars/cloudFunctionCICD.groovy` +- **Issue**: Just prints "Hello World" +- **Impact**: Feature not functional +- **Fix**: Complete implementation or remove + +### 3. **Generic Exception Handling** 🟡 +- **Throughout codebase** +- **Issue**: Catching generic `Exception` everywhere +- **Impact**: Hides specific errors, makes debugging difficult +- **Fix**: Catch specific exceptions + +### 4. **Inconsistent Null Checking** 🟡 +- **Throughout codebase** +- **Issue**: Mix of `== null`, `!= null`, and no safe navigation +- **Fix**: Use Groovy's safe navigation operator (`?.`) + +## Architectural Improvements + +### 1. **No Test Coverage** 🔴 +- **Issue**: No unit or integration tests +- **Impact**: High risk of regressions +- **Fix**: Implement comprehensive test suite + +### 2. **Monolithic Python Script** 🟡 +- **File**: `resources/com/meesho/validate_configs.py` +- **Issue**: 1207 lines in single file +- **Fix**: Refactor into modules + +### 3. **Hard-coded Values** 🟡 +- **Throughout codebase** +- **Issues**: + - GitHub org "Meesho" hard-coded + - Node names like "slave02" + - Various URLs and endpoints +- **Fix**: Move to configuration + +### 4. **No Retry Mechanism** 🟡 +- **Issue**: Retry mechanisms are inconsistently applied - present in ArgoCD sync (--http-retry-max 3) and Docker push (retryDockerPush) but absent from most other critical operations. +- **Impact**: Transient failures cause pipeline failures +- **Fix**: Implement retry with exponential backoff + +### 5. **Inconsistent Logging** 🟡 +- **Issue**: Mix of `log.info()`, `echo`, and print statements +- **Fix**: Standardize logging approach + +## Technical Debt + +### 1. **Commented Code** 🟡 +- **Throughout codebase** +- **Issue**: Large blocks of commented code +- **Fix**: Remove or document why it's kept + +### 2. **TODO Comments** 🟡 +- **Multiple files** +- **Issue**: TODO comments without action plans +- **Fix**: Create tickets or implement + +### 3. **Inconsistent Error Handling** 🟡 +- **Issue**: Some methods set `env.msg`, others don't +- **Fix**: Standardize error handling pattern + +## Recommendations + +### Immediate Actions (P0) +1. Fix the hard-coded branch name bug in buildGradle.groovy +2. Fix method name typo in checkOut.groovy +3. Fix exception typo in helmGenerator.groovy +4. Address security vulnerabilities (passwords, SSH keys) +5. Re-enable or properly remove security scanning + +### Short-term (P1) +1. Fix XML query usage on Gradle files +2. Complete JDK version handling +3. Remove duplicate map keys +4. Implement proper credential handling +5. Add safety checks to file deletion commands + +### Medium-term (P2) +1. Implement comprehensive test coverage +2. Refactor large Python script +3. Create configuration management system +4. Standardize error handling and logging +5. Implement retry mechanisms + +### Long-term (P3) +1. Remove all hard-coded values +2. Clean up technical debt (commented code, TODOs) +3. Implement proper monitoring and alerting +4. Create comprehensive documentation +5. Consider migrating to more modern CI/CD patterns + +## Conclusion + +The codebase requires significant attention to address critical bugs and security vulnerabilities. While functional, it shows signs of organic growth without proper architecture governance. Implementing the recommended fixes will greatly improve reliability, security, and maintainability of the DevOps library. \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ee81bb6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,156 @@ + + +# devops-lib — Claude guide + +Jenkins shared library that backs every Meesho service's CI/CD pipeline. +Loaded by Jenkins as a Global Library; consumed by service repos through a +`@Library('devops-lib')` import in their `Jenkinsfile`. There is **no local +build, test, or run command** — the library only executes inside Jenkins. + +## What this repo is + +- Code is **Groovy** (Jenkins CPS-style, with `@NonCPS` islands). +- `vars/*.groovy` are Jenkins shared-library **globals** — the entry points + service repos call (e.g. `buildPipeline { ... }`, `eksCICD { ... }`). +- `src/com/meesho/stages/**.groovy` — stage implementations dispatched from + the globals (`buildMaven`, `buildGo`, `deployArgoCD`, `notify`, …). +- `src/com/meesho/utilities/**.groovy` — shared helpers (`constructParam`, + `gitActions`, `nodePoolSelection`, `constructTemplate`). +- `resources/com/meesho/**` — per-language `Dockerfile`, `*-deployment.yaml`, + `*-values.yaml` templates rendered into the service repo at deploy time. +- `resources/org/meesho/**-pod.yaml` — Jenkins build-agent pod templates + selected by `env.INFRA_ENV` (`dev` / `stg` / `prd`). + +## Architecture + +Long-form architecture and data flow lives in +[`docs/architecture.md`](docs/architecture.md). Read it before changing any +file under `src/com/meesho/stages/` or `vars/`. + +Pre-existing context worth reading: + +- [`docs/acronyms.md`](docs/acronyms.md) — repo-specific acronyms (BU, + BUILDKIT, CAC, GCPP, INFRA, …). +- [`docs/tribal-knowledge.md`](docs/tribal-knowledge.md) — load-bearing + non-obvious conventions (whitelist git clones, JVM memory auto-calc, canary + enforcement, `ringmaster-bot` user-id switch, etc.). +- [`review-learnings.md`](review-learnings.md) — PR-review-derived rules, + some graduated into the NEVER DO section below. +- [`BUGS_AND_IMPROVEMENTS_REPORT.md`](BUGS_AND_IMPROVEMENTS_REPORT.md) — + catalogue of known bugs / debt; do not silently "fix" these without + reading the linked PR history. + +## Quick reference + +| Task | Command | +|---|---| +| Sanity-check Groovy syntax | `groovy -e 'load "vars/buildPipeline.groovy"'` (not wired in CI; use only as a local lint) | +| Validate Helm value templates | runs inside Jenkins via `validate_configs.py` (no standalone CLI) | +| Run the library | Cannot run locally — push a branch, point a Jenkins job at `@Library('devops-lib@')`, trigger from a service repo | + +**No `Makefile`, no `build.gradle`, no `pom.xml`, no `Dockerfile`, no +`Jenkinsfile`, no `package.json`, no `.github/workflows/` at the repo root.** +A Jenkins job test-loads this library; there is no project-local equivalent. + +**No test suite.** `BUGS_AND_IMPROVEMENTS_REPORT.md` flags this as a known +high-priority gap. Do not fabricate `./gradlew test` or similar — they do not +exist. + +## Stack & tools + +- **Groovy** — Jenkins CPS engine. Beware: any `@NonCPS` method must not be + called across a serialisation boundary (e.g., inside a `parallel` closure) + without the closure itself being `@NonCPS`. +- **Jenkins shared-library layout** — `vars/.groovy` exposes `` + as a global step; `src/com/meesho/**.groovy` is classpath-loaded. +- **Helm / ArgoCD** — service deploys go through + `deployArgoCD.groovy` → `update_helm_repo` → `refresh_app_of_apps` → + `refresh_and_sync`. Step order is load-bearing + ([tribal-knowledge §10](docs/tribal-knowledge.md)). +- **Turbo-Turtle / Ringmaster** — `deployRingmaster.groovy` routes callbacks + based purely on `getCause(UserIdCause).getUserId() == "ringmaster-bot"`. + Renaming that user id silently breaks the routing. +- **Validation** — `resources/com/meesho/validate_configs.py` (1207 lines) is + a monolithic Python validator invoked from Groovy via `sh`. +- **Whitelists** — `getWhitelistedRepos()` does a fresh `git clone` of + `Meesho/whitelists` **on every call** (no caching). Five whitelist gates + each clone independently — this is intentional, do not refactor. + +## Critical conventions + +1. **`vars/` is the public API.** Adding a new entry point means adding a new + file under `vars/`. Renaming or removing a `vars/` global is a breaking + change for every service Jenkinsfile. +2. **`build_tool` switch lives in `src/com/meesho/stages/buildObjHelper.groovy`.** + Supported values today are `maven`, `gradle`, `docker`, `python-*`, + `node-*`, `go*`, `php`. There is no `sbt`, no `rust`. Unknown values fall + through to `defaultBuild` silently. +3. **`@NonCPS` rule.** `constructTemplate._construct()` is annotated + `@NonCPS` because it uses `groovy.text.SimpleTemplateEngine` (Java, + non-serialisable). Keep it `@NonCPS` and call it from a CPS-safe wrapper. +4. **JVM memory auto-calc.** `update_helm_repo` derives `xms == xmx == + memory_limit * 0.5`. Do not hard-code `-Xmx` in `JAVA_OPTS`. +5. **Canary is mandatory for `priority_v2: sp0` / `up0` in prd.** No + whitelist, no bypass. The build fails fast before the Helm update. +6. **PR vs branch detection.** Use `env.CHANGE_ID` (set by the GitHub Branch + Source plugin), never `env.BRANCH_NAME =~ /PR-/`. +7. **Hard-coded `'Meesho'` org.** Many helpers in `gitActions.groovy` / + `constructParam.groovy` embed the GitHub org name as a literal — keep this + stable. Do not parametrise without a coordinated rollout. +8. **Pod selection.** `env.INFRA_ENV` selects the Jenkins agent pod template + via `libraryResource("org/meesho/${env.INFRA_ENV}-pod.yaml")`. Don't add + inline `podTemplate` blocks — they bypass the central agent inventory. + +## NEVER DO + +These rules come from PR-review history (`review-learnings.md`) and +post-incident notes (`docs/tribal-knowledge.md`). Each links to its source. + +- **Never refactor `getWhitelistedRepos` to cache clones across calls** + without confirming the freshness guarantee is no longer required + ([tribal-knowledge §1](docs/tribal-knowledge.md)). +- **Never rename `"ringmaster-bot"`** in `deployRingmaster.groovy` without + coordinating with the Ringmaster team — it's a load-bearing string + ([tribal-knowledge §2](docs/tribal-knowledge.md)). +- **Never set `xms` / `xmx` manually in service Helm values** — the pipeline + computes them from `memory_limit`; manual values collide + ([tribal-knowledge §4](docs/tribal-knowledge.md)). +- **Never print or `cat` an SSH private key to stdout / logs** + (review-learnings `P0_NO_PRIVATE_KEY_LEAK`, PR #634). Always + `withCredentials { ... }` and write to a 0600 file. +- **Never hard-code bare IPs as curl/HTTP targets in pipeline code.** + (review-learnings `P0_HARDCODED_IP_IN_PIPELINE`, PR #681; known existing + exception: `securityScan.groovy:11` flagged for remediation.) Use + DNS-resolvable hostnames. +- **Never bypass canary on `sp0`/`up0` prd deploys** + ([tribal-knowledge §5](docs/tribal-knowledge.md)). +- **Never inline the Turbo-Turtle JSON payload in `curl` differently from + the current `-d '$newCICD_JSON'` pattern** without re-verifying bash + escaping ([tribal-knowledge §6](docs/tribal-knowledge.md)). +- **Never use `--no-verify` on `git commit`** — pre-commit hooks include + TruffleHog secret scanning and CAC validation. Bypassing is a P0 + policy violation. +- **Never commit to `main` or `develop` directly** — those branches are + protected; all changes go via PR. + +## Where to ask + +- `#ci-cd-status` Slack channel — pipeline failures, library questions. +- `@maintainer` Slack handle is required in every pipeline `param` map + (consumed by `src/com/meesho/stages/notify.groovy:9`). + +## Pointers + +- [docs/architecture.md](docs/architecture.md) — full module-by-module map. +- [docs/acronyms.md](docs/acronyms.md) — repo-specific abbreviations. +- [docs/tribal-knowledge.md](docs/tribal-knowledge.md) — non-obvious patterns. +- [review-learnings.md](review-learnings.md) — PR-review-derived rules, + graduation candidates for this file's NEVER DO section. +- [review.md](review.md) — review process / rubric. +- [BUGS_AND_IMPROVEMENTS_REPORT.md](BUGS_AND_IMPROVEMENTS_REPORT.md) — known + bug catalogue; check before "fixing" anything that looks suspect. diff --git a/README.md b/README.md index 71f2965..b252434 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,67 @@ # devops-lib +--- + +## Parameters +Most of the functionality depends on the parameters provided by the users in form of groovy map of key and value pairs. The supported parameters are as below: + +### Required parameters +**repo_name**: The key repo_name is required for checking out the code in a subdirectory. The value is the repository name that you want to checkout + +**build_tool**: This parameter is required to identify which build_tool to use in the pipeline. The supported values are *maven*, *gradle*, *docker*, *python*, *node*, *go*, *php* (and their prefixed variants such as *maven-3.3-jdk-17*, *python-3*, *node-16*, *go1.21*) + +**maintainer** : This parameter is required to send the notification in the slack channel *#ci-cd-status*. Please provide your slack username here + +### Optional parameter + +`devops-lib` is Meesho's Jenkins Shared Library that provides a unified CI/CD pipeline for all microservices across the organisation. Consumer repos load it via `@Library('devops-lib@main')` and call a single `eksCICD(repo)` entry point — the library handles language-specific building (Maven, Go, Gradle, Node.js, Python, PHP), code quality gates (Sonar), Docker image publishing to GAR/ECR, Helm chart updates, and ArgoCD-based deployment to GKE/EKS clusters. Build status and deployment metadata are reported back to Ringmaster and Slack. + +**Stack:** Groovy (Jenkins Shared Library) · ArgoCD · Helm · GCP (GKE, GAR, GCS, Vault, Sonar) · AWS (EKS, ECR, S3) + +## Dependencies + +**push_to_jfrog**: By default master, main, gcp-main, and gcp-master branches push artifacts to jfrog/s3 repository, set this parameter to true to push artifacts from non-master branches + +--- + +## config.yaml schema (consumer services) + +Every service that uses this library must provide a `config.yaml`: + +| Key | Required | Description | +|-----|----------|-------------| +| `repo_name` | yes | GitHub repo slug — must match exactly | +| `build_tool` | yes | `maven`, `go`, `gradle`, `node-*`, `python-*`, `php`, `docker` | +| `dockerBuildVersion` | yes | Drives Dockerfile template: `maven-21`, `go-1.22`, `node-20`, etc. | +| `team` | yes | Team slug — validated against `buTeamMapping` | +| `bu` | yes | Business unit: `supply`, `demand`, `central`, `dataengg`, `datascience`, `mcache`, `infra` | +| `maintainer` | yes | GitHub handle for Slack notifications | +| `deployment_order` | yes | List of ArgoCD application names to deploy | +| `notify_channel` | no | Slack channel (default: `ci-cd-status`) | +| `skip_sonar` | no | Whitelist-gated; see `constructParam.groovy` | +| `deployArgo` | no | Set `false` to skip ArgoCD sync | +| `appConfigEnabled` | no | Required `true` for `stg`; whitelist-gated | +| `skip_test` | no | Skip unit tests (Maven) | +| `push_to_jfrog` | no | Publish JAR to JFrog Artifactory | +| `push_to_s3` | no | Push artifact to S3 | +| `build_packages` | no | System development packages required while compiling (currently consumed by Rust builds; for example `libpq-dev`) | +| `runtime_packages` | no | System runtime libraries required by the compiled binary (currently consumed by Rust builds; for example `libpq5`) | + +## Adding this library to a new service + +```groovy +// Jenkinsfile +@Library('devops-lib@main') _ + +eksCICD([ + repo_name: 'my-service' +]) +``` + +Place `config.yaml` at the repo root with the required fields above. + +## Adding a new build stage + +1. Create `src/com/meesho/stages/build.groovy` implementing `def run(Map config)`. +2. Add a `case` in `src/com/meesho/stages/buildObjHelper.groovy`. +3. Add a Dockerfile template in `resources/com/meesho/-Dockerfile` if needed. diff --git a/ai-blitz/README.md b/ai-blitz/README.md new file mode 100644 index 0000000..f7c13e8 --- /dev/null +++ b/ai-blitz/README.md @@ -0,0 +1,45 @@ +# AI Blitz — devops-lib task catalogue + +Tasks identified for the AI Blitz Week 3 capstone (Task 11 from the Week 3 playbook). Each task is a small, agent-doable unit of work grounded in real findings from `BUGS_AND_IMPROVEMENTS_REPORT.md`, `docs/tribal-knowledge.md`, and `review-learnings.md`. + +## Mix + +5 tasks: 3 small features + 2 bug fixes. + +| # | Task | Type | Files touched (rough) | Difficulty | +|---|---|---|---|---| +| 01 | [`task-01-securityScan-hardcoded-ip.md`](task-01-securityScan-hardcoded-ip.md) | Bug fix (P0 security) | 2 | low | +| 02 | [`task-02-checkoutSubmodule-typo.md`](task-02-checkoutSubmodule-typo.md) | Bug fix (correctness) | 1-2 + consumer repos | medium | +| 03 | [`task-03-onlyPushtoJfrog-jdk-versions.md`](task-03-onlyPushtoJfrog-jdk-versions.md) | Small feature | 1 | low–medium | +| 04 | [`task-04-buildPipeline-agent-label-param.md`](task-04-buildPipeline-agent-label-param.md) | Small feature | 3 | low | +| 05 | [`task-05-cloudFunctionCICD-implementation.md`](task-05-cloudFunctionCICD-implementation.md) | Small feature | 4-5 + 1 consumer | high | + +## Capstone picks + +| Day | Mode | Task | Why this pick | +|---|---|---|---| +| Day 4 (Thu) | Interactive (war-room) | **Task 05 — Implement `cloudFunctionCICD`** | Real design decisions throughout (gen-1 vs gen-2 syntax, runtime → buildObjHelper mapping, service-account auth path). Maximises learning signal when humans correct the agent in real time. | +| Day 5 (Fri) | Autonomous | **Task 02 — Fix `chekoutSubmodule` typo** | Clear acceptance criteria, mostly mechanical, the deprecated-alias-with-log-warning pattern is recognisable. Agent can solo without supervision. | + +## Adding a new task + +1. Copy [`_TEMPLATE.md`](_TEMPLATE.md) to `task-NN-.md`. +2. Number sequentially (next is 06). +3. Fill out Goal + Acceptance Criteria. Cite a source — `BUGS_AND_IMPROVEMENTS_REPORT.md` §, `docs/tribal-knowledge.md` §, or a specific `review-learnings.md` rule ID. +4. Update the table above. + +## Source documents the catalogue draws from + +- [`../BUGS_AND_IMPROVEMENTS_REPORT.md`](../BUGS_AND_IMPROVEMENTS_REPORT.md) — known bugs + tech debt +- [`../docs/tribal-knowledge.md`](../docs/tribal-knowledge.md) — non-obvious conventions +- [`../review-learnings.md`](../review-learnings.md) — PR-review-derived rules +- [`../docs/architecture.md`](../docs/architecture.md) — module map, downstream services, invariants + +## Out of scope + +Tasks **not** in this catalogue (good candidates, but not picked): + +- Full test-harness bootstrap (JenkinsPipelineUnit) + first unit tests — large scaffolding effort, deferred to a dedicated PR. +- ArgoCD `--password-stdin` refactor — pure refactor, no behavioural change; lower learning value. +- Refactor `validate_configs.py` (1207 lines) into modules — too large for a single Blitz task. +- `xq` (XML query) → Groovy parser on Gradle files in `buildGradle.groovy` — same reason. diff --git a/ai-blitz/_TEMPLATE.md b/ai-blitz/_TEMPLATE.md new file mode 100644 index 0000000..8020aba --- /dev/null +++ b/ai-blitz/_TEMPLATE.md @@ -0,0 +1,20 @@ +# Task NN: + +- **Type:** Feature / Bug fix / Refactor / Test gap +- **Source:** `<file>.md` §<section> or `review-learnings.md` `<RULE_ID>` + +## Goal + +One sentence: what should be true after this task is done. + +## Acceptance Criteria + +- [ ] Observable behaviour 1 +- [ ] Observable behaviour 2 +- [ ] Tests added/updated covering the new behaviour (skip if devops-lib has no test suite — note that here) +- [ ] All existing tests pass (or: smoke-test via `@Library('devops-lib@<branch>')` from a sandbox consumer Jenkinsfile) +- [ ] Docs updated if the task changes a documented behaviour (`README.md`, `CLAUDE.md`, `docs/tribal-knowledge.md`, `docs/architecture.md`, the relevant `review-learnings.md` rule) + +## Notes / known gotchas + +(Optional — surface design decisions or constraints the agent should know upfront.) diff --git a/ai-blitz/task-01-securityScan-hardcoded-ip.md b/ai-blitz/task-01-securityScan-hardcoded-ip.md new file mode 100644 index 0000000..6f3a40a --- /dev/null +++ b/ai-blitz/task-01-securityScan-hardcoded-ip.md @@ -0,0 +1,22 @@ +# Task 01: Replace hard-coded IP in `securityScan.groovy` with DNS hostname + +- **Type:** Bug fix (security, P0) +- **Source:** `BUGS_AND_IMPROVEMENTS_REPORT.md` §2.1 + `review-learnings.md` rule `P0_HARDCODED_IP_IN_PIPELINE` + +## Goal + +`src/com/meesho/stages/securityScan.groovy` no longer references a bare IP for the scanner endpoint; the `P0_HARDCODED_IP_IN_PIPELINE` rule in `review-learnings.md` no longer needs the "known exception" carve-out for this file. + +## Acceptance Criteria + +- [ ] `securityScan.groovy:11` uses a DNS hostname (e.g. `security-scan.meeshogcp.in` or whatever DevOps allocates) instead of `172.31.5.29:63232` +- [ ] If no DNS exists yet, the task surfaces a DevOps ask before merging — does **not** ship with a placeholder IP +- [ ] `grep -rE '\b(172|10|192)\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+' src/com/meesho/ vars/` returns zero hits for HTTP/curl targets +- [ ] `review-learnings.md` rule `P0_HARDCODED_IP_IN_PIPELINE` has its "known exception: securityScan.groovy:11 uses 172.31.5.29:63232 pending remediation" carve-out **removed** in the same PR +- [ ] No retry / behavioural change — same POST shape, single call, only the hostname changes +- [ ] Smoke-test via a Jenkins job pointed at `@Library('devops-lib@<branch>')` confirming the scan POST succeeds against the new DNS + +## Notes / known gotchas + +- The receiver is a Meesho-internal scanner — coordinate with the security team to confirm the DNS name and that it's reachable from the Jenkins agent pod's network policy. +- Don't add a fallback to the IP "just in case" — the whole point of the rule is to fail loud on DNS issues, not to silently drop back to a hardcoded address. diff --git a/ai-blitz/task-02-checkoutSubmodule-typo.md b/ai-blitz/task-02-checkoutSubmodule-typo.md new file mode 100644 index 0000000..55587be --- /dev/null +++ b/ai-blitz/task-02-checkoutSubmodule-typo.md @@ -0,0 +1,30 @@ +# Task 02: Fix `chekoutSubmodule` method-name typo across all call sites + +- **Type:** Bug fix (correctness) +- **Source:** `BUGS_AND_IMPROVEMENTS_REPORT.md` §1.2 + +## Goal + +Method name in `src/com/meesho/stages/checkOut.groovy:20` reads correctly as `checkoutSubmodule`; every caller in the codebase resolves; consumer Jenkinsfiles that reference the old name don't break overnight. + +## Acceptance Criteria + +- [ ] `checkOut.groovy:20` defines `def checkoutSubmodule(String repo_name)` — not `chekoutSubmodule` +- [ ] Every `grep -rn 'chekoutSubmodule'` hit in `src/`, `vars/`, `resources/` is updated to `checkoutSubmodule` +- [ ] Cross-org sweep: search `Meesho/*` for `chekoutSubmodule` references and either: + - (a) raise companion PRs on each consumer to update the call, **or** + - (b) add a backwards-compat alias in `checkOut.groovy`: + ```groovy + def chekoutSubmodule(String repo_name) { + log.warning('chekoutSubmodule is deprecated — use checkoutSubmodule') + return checkoutSubmodule(repo_name) + } + ``` +- [ ] Alias (if present) is documented in a `CHANGELOG`-style note with a target removal date +- [ ] Smoke-test via a consumer Jenkinsfile that exercises the submodule checkout path +- [ ] No behavioural change in the underlying method body — only the name changes + +## Notes / known gotchas + +- The alias path is the safer choice in a shared library — consumer Jenkinsfiles get reviewed and merged on their own schedule, and a hard rename will silently fail across the org. +- Once the alias is in, run `gh search code 'chekoutSubmodule' --owner Meesho --json repository,path` periodically to track removal readiness. diff --git a/ai-blitz/task-03-onlyPushtoJfrog-jdk-versions.md b/ai-blitz/task-03-onlyPushtoJfrog-jdk-versions.md new file mode 100644 index 0000000..e7eddd4 --- /dev/null +++ b/ai-blitz/task-03-onlyPushtoJfrog-jdk-versions.md @@ -0,0 +1,22 @@ +# Task 03: Complete JDK version handling in `onlyPushtoJfrog.groovy` + +- **Type:** Small feature +- **Source:** `BUGS_AND_IMPROVEMENTS_REPORT.md` §1.5 + +## Goal + +When a user picks JDK 11 or JDK 21 in the `onlyPushtoJfrog` choice parameter, the job runs under that JDK — not the JDK 8 fall-through default. + +## Acceptance Criteria + +- [ ] `vars/onlyPushtoJfrog.groovy:16` choice parameter remains `['jdk8', 'jdk11', 'jdk17', 'jdk21']` (no values removed) +- [ ] The if/else at `vars/onlyPushtoJfrog.groovy:48-53` handles **all four** JDK selections, each setting the correct `JAVA_HOME` path that exists in the build-tools pod image +- [ ] An unknown JDK selection (shouldn't happen via the choice param, but defensive) sets `JAVA_HOME` to a fail-fast value and logs `log.error("Unknown jdk_version: ${jdk_version}")` instead of silently defaulting to JDK 8 +- [ ] Verify the JDK paths against the actual `devops-tools:lunar-vX.Y.Z` image used by `prd-pod.yaml` / `stg-pod.yaml`. If `jdk11` or `jdk21` aren't in the image, raise a `tribal-knowledge §12` follow-up (bump the build-tools image tag) **before** this task can complete +- [ ] Smoke-test all 4 JDK selections via a sandbox Jenkins job — confirm `java -version` reports the expected JDK in each run +- [ ] No behavioural change for `jdk8` / `jdk17` selections — those branches stay identical + +## Notes / known gotchas + +- The fall-through bug is silent: builds "succeed" with the wrong JDK, which can mask bytecode-version incompatibilities or break records of which JDK actually built the artifact. The fail-fast on unknown values is intentional. +- `tribal-knowledge.md` §12 calls out that toolchain binaries belong in the pod image, not curled at build time — keep this task aligned with that convention. diff --git a/ai-blitz/task-04-buildPipeline-agent-label-param.md b/ai-blitz/task-04-buildPipeline-agent-label-param.md new file mode 100644 index 0000000..7e23030 --- /dev/null +++ b/ai-blitz/task-04-buildPipeline-agent-label-param.md @@ -0,0 +1,26 @@ +# Task 04: Make Jenkins agent label configurable via `agent_label` parameter + +- **Type:** Small feature +- **Source:** `BUGS_AND_IMPROVEMENTS_REPORT.md` §3 (hard-coded `node('slave02')` in `vars/buildPipeline.groovy:16`) + +## Goal + +`buildPipeline { ... }` consumers can override the Jenkins agent node label without forking the library; existing consumers (no override) still pin to `slave02` so nothing breaks. + +## Acceptance Criteria + +- [ ] `vars/buildPipeline.groovy:16` reads `param.agent_label` and falls back to `'slave02'` when unset: + ```groovy + def agentLabel = param.agent_label ?: 'slave02' + node(agentLabel) { ... } + ``` +- [ ] `README.md` `## Optional parameter` section documents `agent_label` with the same shape as the other params (key, type, default, description). Default must be the legacy `slave02` for backwards compatibility +- [ ] A consumer Jenkinsfile that passes `agent_label: 'gke-build-agent'` runs on that node; one that omits it still runs on `slave02` +- [ ] `grep -rn "node\('slave02'\)" vars/ src/` shows zero hard-coded uses **other than the fallback string in `buildPipeline.groovy`**. If other files hard-code the same node, list them out and either fix in the same PR or open follow-up tasks +- [ ] Smoke-test: one PR pipeline with the override, one without +- [ ] Add a `## NEVER DO` entry to `CLAUDE.md`: *"Don't add new `node('<literal>')` blocks in `vars/`. New entry points must accept `agent_label` from the param map and fall back to `'slave02'` or a documented default."* + +## Notes / known gotchas + +- `eksCICD.groovy` and `gkeCICD.groovy` use `podTemplate(yaml: libraryResource("org/meesho/${env.INFRA_ENV}-pod.yaml"))` instead of a `node()` block — they're outside the scope of this task. Only `buildPipeline.groovy` (and any other `node('<literal>')` callers grep finds) needs the change. +- Don't change the default to `null` or empty — services in the org currently rely on the implicit `slave02` pin; flipping the default mid-flight will silently move builds to whichever agent the master picks. diff --git a/ai-blitz/task-05-cloudFunctionCICD-implementation.md b/ai-blitz/task-05-cloudFunctionCICD-implementation.md new file mode 100644 index 0000000..f3bd67d --- /dev/null +++ b/ai-blitz/task-05-cloudFunctionCICD-implementation.md @@ -0,0 +1,40 @@ +# Task 05: Implement the `cloudFunctionCICD` skeleton + +- **Type:** Small feature +- **Source:** `BUGS_AND_IMPROVEMENTS_REPORT.md` §1.6 (`cloudFunctionCICDFlow()` is currently a stub containing only `sh 'ls -al'; echo 'Hello World'`) + +## Goal + +`vars/cloudFunctionCICD.groovy` actually deploys a GCP Cloud Function. Services that use `cloudFunctionCICD { ... }` get a working pipeline instead of the current no-op skeleton. + +## Acceptance Criteria + +- [ ] `vars/cloudFunctionCICD.groovy` exposes `def call(Map param)` (not the current zero-arg `cloudFunctionCICDFlow()`), accepting at minimum: + - `repo_name` + - `function_name` + - `runtime` — one of `nodejs20` / `python311` / `go121` / etc. + - `entry_point` + - `region` + - `service_account` + - `maintainer` +- [ ] Stages run in this order: + 1. `checkOut` (reuse `src/com/meesho/stages/checkOut.groovy`) + 2. Optional: `buildObjHelper.run(param.runtime)` if the runtime needs a transpile/install step (e.g. `node-*` → `npm install && npm run build`). Else skip. + 3. `gcloud functions deploy ${function_name} --gen2 --runtime ${runtime} --entry-point ${entry_point} --region ${region} --service-account ${service_account} --source .` + 4. `notify` (reuse `src/com/meesho/stages/notify.groovy`) — same Slack format as `buildPipeline` +- [ ] Pod selection uses `libraryResource("org/meesho/${env.INFRA_ENV}-pod.yaml")` — never an inline `podTemplate` +- [ ] Auth: `gcloud` uses ambient Workload Identity via the pod's `service_account`; **no key files written to disk** +- [ ] Branch gating: only deploy from `master`, `main`, `gcp-main`, or `gcp-master` (match the JFrog/S3 gate in `buildMaven.groovy`) unless `param.deploy_from_any_branch == true` +- [ ] Deployment-tracker callback fires on success/failure (reuse `notify.groovy:postTrackingApi`) +- [ ] One real consumer service uses the new entry point end-to-end as the smoke test +- [ ] `README.md` `## Adding this library to a new service` gains a Cloud Function example block alongside the existing Jenkinsfile sample +- [ ] `docs/architecture.md` § "Entry points (vars/)" updated — `cloudFunctionCICD` is no longer flagged as "stub" +- [ ] `BUGS_AND_IMPROVEMENTS_REPORT.md` §1.6 entry is moved to a Resolved section (or struck-through per repo convention) +- [ ] `docs/index.md` / `docs/wiki/index.md` updated if the entry point gets new dedicated documentation + +## Notes / known gotchas + +- Use `gcloud functions deploy --gen2` explicitly — gen-1 syntax differs (no `--source .`, different IAM model). Don't write code that works on whichever version happens to be default in the build-tools image. +- The user-authorization gate from `eksCICD.groovy:12-26` (allowed-users list, Ringmaster banner) is intentionally **not** copied to this entry point unless DevOps says otherwise. Cloud Functions are deployed less frequently; the gate may be overkill. +- If `runtime` doesn't match a known `buildObjHelper` case (e.g. `python311` vs `python-3.11`), prefer extending the `buildObjHelper` switch to recognise the GCP runtime naming rather than inlining build logic in `cloudFunctionCICD`. +- This task touches a lot of files (vars/, README, architecture.md, BUGS report) — chunk into reviewable commits if the agent is going autonomous, otherwise one PR is fine. diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..d25daf0 --- /dev/null +++ b/build.gradle @@ -0,0 +1,38 @@ +plugins { + id 'groovy' +} + +repositories { + maven { url 'https://repo.jenkins-ci.org/releases/' } + maven { url 'https://repo.jenkins-ci.org/public/' } + mavenCentral() +} + +sourceSets { + // Main sources are Jenkins Shared Library scripts loaded at runtime by JenkinsPipelineUnit. + // They depend on Jenkins API and are not pre-compiled — loadScript() handles them at test time. + main { + groovy { srcDirs = [] } + java { srcDirs = [] } + } + test { + // test/unit — actual test classes + // test/stubs — minimal stub implementations of Jenkins stage classes used by dispatch tests + groovy { srcDirs = ['test/unit', 'test/stubs'] } + } +} + +dependencies { + testImplementation 'com.lesfurets:jenkins-pipeline-unit:1.22' + // Match the Groovy version bundled by JenkinsPipelineUnit + testImplementation 'org.codehaus.groovy:groovy-all:2.4.21' + testImplementation 'junit:junit:4.13.2' +} + +test { + systemProperty 'user.dir', rootDir.absolutePath + testLogging { + events 'passed', 'skipped', 'failed' + exceptionFormat 'full' + } +} diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 0000000..0409877 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,262 @@ +# SECURITY.md +<!-- Auto-generated by /m-docs:security-init. Edit freely — re-running preserves your changes. --> + +> **Scope:** devops-lib is a Jenkins Shared Library, not a deployable web service. It has no HTTP endpoints of its own. This document covers the security properties of the library's CI/CD execution: how secrets are handled, what trust boundaries exist, what security rules new code must follow, and known gaps. + +--- + +## Authentication + +devops-lib has no user-facing HTTP endpoints and performs no JWT or session validation. Authentication is enforced at two points: + +### Build trigger gate +`vars/eksCICD.groovy:12–26` validates that every Jenkins build is triggered by an authorized caller: +- `ringmaster-bot` — Ringmaster's automated trigger +- `turbo-turtle` — Turbo-Turtle's CI callback trigger +- `allowedUsers` — a hardcoded list of DevOps engineer email addresses for emergency access + +Unauthorized triggers are hard-rejected before any pipeline logic runs. + +### Outbound credential injection +All outbound API calls use Jenkins' `withCredentials` binding — credentials are never hardcoded in source. Jenkins masks bound variables in console output automatically. Key credential IDs: + +| Credential ID | Used for | Scope | +|---|---|---| +| `cicd-github-app` | Cloning `Meesho/whitelists`, `devops-argo-config`, `devops-helm-charts` | All builds | +| `svc-devops-meesho` | GitHub API, JFrog Artifactory | Build + deploy | +| `ringmaster-token` | Ringmaster callback API | Notify stage | +| `argocd-{bu}-prd-creds` / `argocd-dev-creds` | ArgoCD CLI login | Deploy stage | +| `vault-prd-token` / `vault-dev-token` | Vault secret fetch | Node builds only | +| `sonar-token-prod` / `sonar-token-{bu}-dev` | SonarQube analysis | Build stages | + +--- + +## Trust Boundaries + +### Build trigger trust boundary + +| Layer | What happens | Where | Confidence | +|---|---|---|---| +| Ringmaster / Turbo-Turtle | Validates human approval, triggers Jenkins build | Upstream (Ringmaster infra) | docs-referenced | +| devops-lib `eksCICD` | Validates trigger source against `allowedUsers` list | `vars/eksCICD.groovy:12` | code-confirmed | +| Jenkins pipeline | Executes build stages with injected credentials | Jenkins agents (GKE pods) | code-confirmed | + +**Trust assumption:** devops-lib assumes that any build triggered by `ringmaster-bot` or `turbo-turtle` has already been approved by the Ringmaster/Turbo-Turtle authorization flow. It does NOT re-validate the approval — it trusts the trigger identity. + +### Supply chain trust boundary (CRITICAL) + +`devops-lib@main` is loaded via `@Library('devops-lib@main')` by every Meesho microservice on every build. **A malicious or buggy merge to `main` is an immediate supply chain attack on all 100+ consumer services' CI/CD pipelines.** + +- Whoever can merge to `devops-lib@main` controls the full CI/CD path for all Meesho microservices +- Branch protection rules for `main` are enforced at the GitHub repository level (not visible in this repo's source) +- No `.github/CODEOWNERS` file is present in the repository + +<!-- TODO: Confirm that devops-lib's main branch has required PR reviews and no direct push access for non-DevOps-leads. This is the highest-risk trust boundary in the library. --> + +### Whitelist repo trust boundary + +Policy exceptions (sonar skip, multizone, AppConfig, CAC) are fetched from `Meesho/whitelists` at build time via `cicd-github-app` credential. If `Meesho/whitelists` is compromised or the `cicd-github-app` credential is stolen, an attacker could: +- Add any service to `skip-sonar-whitelist` to bypass quality gates +- Add a service to `multizone-enabled-repos` to block its deployments +- Remove a service from `ValidateCacConfig` to bypass config validation + +### Jenkins agent trust boundary + +Build stages execute in GKE pods (see `resources/org/meesho/prd-pod.yaml`). Secrets injected via `withCredentials` exist in the pod's process environment for the duration of the `withCredentials` block and are cleared afterward. `env.VAULT_TOKEN` is temporarily set during Vault secret fetch and immediately cleared: + +```groovy +// buildNode.groovy:548-553 +env.VAULT_TOKEN = "${TOKEN}" +sh(script:"${vault_cmd}") +env.VAULT_TOKEN = 'empty' // ← cleared immediately after use +``` + +**Note:** Assigning to `env.*` persists the value in Jenkins pipeline serialized state for the duration of that block — it is not fully memory-isolated like a `withCredentials` binding. + +### Process / runtime boundaries + +| Boundary | Inside (trusted) | Outside | Crossing mechanism | +|---|---|---|---| +| `withCredentials` block | Jenkins credential binding (secret) | Pipeline Groovy scope | Automatic unset on block exit | +| Jenkins agent pod | Build process, injected creds | Other pods, external network | K8s network policy, GKE service account | +| `set +x` shell guard | ArgoCD password in shell arg | Jenkins console log | `set +x` before credential use in `deployArgoCD.groovy:493` | +| DinD container | Docker daemon | Build container | TCP socket (`dind-prd-svc`) — not Unix socket (avoids privilege escalation) | + +--- + +## Entry Points + +devops-lib has no HTTP entry points. It is invoked as a Jenkins Shared Library. + +### Build trigger (sole entry point) + +| Trigger | Who sends it | Auth check | +|---|---|---| +| Ringmaster-initiated build | `ringmaster-bot` Jenkins user | `allowedUsers` gate in `eksCICD.groovy` | +| Turbo-Turtle-initiated build | `turbo-turtle` Jenkins user | `allowedUsers` gate in `eksCICD.groovy` | +| DevOps engineer direct trigger | Email in `allowedUsers` list | `allowedUsers` gate | +| Unauthorized user | Any other Jenkins user | Hard-rejected — pipeline aborts immediately | + +### Outbound calls (not entry points, but relevant to trust) + +All outbound calls are made FROM Jenkins agents TO external services. See [docs/downstreams.md](downstreams.md) for the full inventory. + +Security additions to downstreams.md: + +| Service | Protocol | Data sent | Risk | Notes | +|---|---|---|---|---| +| ArgoCD | HTTPS + gRPC | App names, image tags | Low | `set +x` guards password in shell | +| Ringmaster | HTTPS | Build result, image tag, repo name, team | Low | Auth via `ringmaster-token` credential | +| Turbo-Turtle | **HTTP** (plain) | Build result, image tag, repo name | Low — accepted risk | Internal VPC only, not reachable externally | +| Deployment Tracker | **HTTP** (plain) | Repo name, deploy timestamp, tag | Low — accepted risk | Internal VPC only, legacy endpoint | +| Security scanner | **HTTP** to `172.31.5.29:63232` | Repo name, branch | Low — accepted risk | Internal scanner, hardcoded IP | +| SonarQube | HTTPS | Source code analysis | Low | Token injected via `withCredentials` | +| Vault | HTTPS | Vault path (not secrets) | Low | Token cleared immediately after fetch | +| GitHub | HTTPS | Git operations | Low | `cicd-github-app` credential | + +--- + +## Authorization + +### Policy enforcement model + +devops-lib enforces policy through two mechanisms: + +1. **Whitelist-controlled gates** — `constructParam.groovy` checks `Meesho/whitelists` at runtime for per-repo exceptions. No service can grant itself a bypass; all exceptions require a PR to `Meesho/whitelists` reviewed by DevOps. + +2. **Library-level enforcement** — `deployArgoCD.groovy` enforces canary for Tier-1 services, `eksCICD.groovy` enforces the trigger gate. These cannot be overridden by service config. + +### allowedUsers list + +The bypass list at `vars/eksCICD.groovy:12` contains hardcoded engineer email addresses. This list has no expiry mechanism — emails remain valid until manually removed. + +<!-- TODO: Confirm that the allowedUsers list is audited periodically to remove email addresses of engineers who have left the organization. --> + +--- + +## Data Classification + +devops-lib handles no end-user PII. All data is build metadata: + +### Non-PII (safe to log and pass to external services) + +| Data | Where it appears | Notes | +|---|---|---| +| `repo_name` | All stages, Ringmaster callback | GitHub org slug — not sensitive | +| `build_tool` | Build stages | Language identifier | +| `cicd_environment` | All stages | prd / stg / int / ftr | +| `TAG` (image tag) | Deploy stages, Slack notifications | `<branch>-<git-sha>` — not sensitive | +| `notify_channel` | Notify stage | Slack channel name | +| `team` / `bu` | Build stages, node pool selection | Org metadata | +| `deployment_order` | ArgoCD deploy | App names in devops-argo-config | +| Build result / duration | Slack, Ringmaster, Turbo-Turtle | Build observability | + +### Sensitive (not PII, but must be handled with care) + +| Data | Where it exists | Handling | +|---|---|---| +| Jenkins credential values | `withCredentials` blocks | Never logged, masked in console | +| Vault secret paths | `buildNode.groovy:536-537` | Path logged (not value); value only in `sh` subprocess | +| `ARGO_USERNAME` / `ARGO_PASSWORD` | `deployArgoCD.groovy:490-494` | `set +x` guard prevents echo in logs | +| `GITHUB_TOKEN` | Git clone operations | Via `gitUsernamePassword` binding — not logged | + +--- + +## Data Lifecycle & Erasure + +No end-user data stored. Build artifacts: + +**Known:** +- Jenkins build logs: retained per Jenkins job configuration (managed by Jenkins admins, not devops-lib) +- Docker images in GAR/ECR: no TTL configured in devops-lib — lifecycle managed by GAR cleanup policies outside this library +- Deployment history in Ringmaster: managed by Ringmaster service + +**Unknown:** Retention policy for build logs and deployment records is not configurable from devops-lib. +<!-- TODO: Confirm Jenkins build log retention policy with the infrastructure team. --> + +--- + +## Data Storage & Encryption + +### At rest +devops-lib has no persistent storage. It reads from GitHub, Jenkins credentials store, and Vault; it writes to GitHub repos (argo-config, helm-charts) and pushes Docker images to GAR/ECR. + +| Store | What is stored | Managed by | +|---|---|---| +| Jenkins credentials store | All CI/CD credentials (tokens, passwords) | Jenkins admins | +| GAR / ECR | Docker images | GCP / AWS infra | +| devops-helm-charts / devops-argo-config | Helm values, ArgoCD manifests | devops-lib writes; git is the store | + +### In transit +- All GitHub API calls: HTTPS ✓ +- ArgoCD CLI: HTTPS + gRPC ✓ +- SonarQube, Vault, Ringmaster API: HTTPS ✓ +- Turbo-Turtle, Deployment Tracker, security scanner: **plain HTTP** — accepted risk (internal VPC, not reachable externally) + +### Secrets management +All secrets are injected at runtime from Jenkins credentials store via `withCredentials`. No secrets in source code, config files, or environment variables baked into the library. Credentials are identified by their Jenkins credential ID (e.g. `ringmaster-token`, `vault-prd-token`) — the actual values are never stored in this repository. + +--- + +## Input Validation + +devops-lib takes inputs from two sources: + +### Service `config.yaml` (primary input) +Read by `getYamlParameter.getParam()`. Fields are used directly without schema validation — devops-lib trusts the config.yaml from the consumer service's own repository (cloned via authenticated git). Malformed configs produce runtime errors, not silent misbehaviour. + +`buTeamMapping.groovy` validates `bu` and `team` fields against a known mapping and throws on invalid values. + +### What's NOT validated + +- **`repo_name`** in config.yaml: used in ArgoCD app names, Slack messages, and Vault paths. Not sanitized against shell injection — passed directly into `sh()` scripts. Risk is mitigated because `repo_name` comes from the service's own config.yaml in its own GitHub repo (already authenticated). +- **`notify_channel`**: passed directly to Slack API — no format validation. A malformed channel name produces a Slack API error, not a security issue. +- **`deployment_order` app names**: passed to `argocd app sync` — no format validation. An invalid app name produces an ArgoCD error. + +--- + +## Security Headers & CORS + +Not applicable — devops-lib has no HTTP server and serves no responses. + +--- + +## Security Rules for New Code + +These rules apply to anyone adding code to devops-lib: + +**Credentials:** All secrets must be injected via `withCredentials` — never assign a credential value to a variable outside a `withCredentials` block, never interpolate credentials into log statements, and never store them in `env.*` variables beyond the immediate operation that needs them. + +**Shell commands with credentials:** Use `set +x` immediately before any `sh()` that includes a credential variable as an argument (as done in `deployArgoCD.groovy:493`). Without `set +x`, Jenkins echoes the full shell command including the credential value to the build log. + +**Policy enforcement:** Never add inline conditionals for repo-level policy exceptions. All exceptions must go through `Meesho/whitelists` — see [ADR-0003](adr/0003-policy-exceptions-in-separate-whitelist-repo.md). + +**Supply chain hygiene:** Any change to `vars/eksCICD.groovy` or `src/com/meesho/utilities/constructParam.groovy` affects every Meesho microservice build. These files require extra scrutiny — treat them as Tier-1 code. + +**DinD image:** The Docker-in-Docker image must come from the internal GAR registry (`asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/docker:28-dind`). Never use `docker:N-dind` from Docker Hub — it is not security-scanned and will be blocked by network policy. (See `docs/tribal-knowledge.md` TK#14.) + +The following security rules are already enforced in CLAUDE.md NEVER DO and are not repeated here: +- **Never hard-code AWS/GCP account IDs, vault tokens, or credentials** — all secrets are passed via Jenkins credentials (`withCredentials`) or injected through `constructParam.run()`. Credentials IDs are defined in `constructParam.groovy`. +- **Never change the string `"ringmaster-bot"`** without coordinating with the Ringmaster team — it is the sole signal that routes callbacks to Ringmaster vs Turbo-Turtle. + +--- + +## Security Debt Tracker + +| ID | Gap | Severity | Source | Ticket | +|---|---|---|---|---| +| SEC-DL-001 | `allowedUsers` list in `eksCICD.groovy` contains engineer email addresses with no expiry — stale access risk if engineers leave the org | Low | Code-discovered | — | +| SEC-DL-002 | No `.github/CODEOWNERS` visible in repo — supply chain protection for `main` branch depends entirely on GitHub repo settings not auditable from source | Medium | Code-discovered | — | +| SEC-DL-003 | Security scanner endpoint hardcoded as `172.31.5.29:63232` (plain HTTP, no auth) — if IP changes, scanner silently stops running | Low | Code-discovered | — | +| SEC-DL-004 | `env.VAULT_TOKEN` temporarily assigned in `buildNode.groovy:550` — token exists in Jenkins pipeline serialized state during Vault fetch block (cleared immediately after) | Low | Code-discovered | — | +| SEC-DL-005 | Turbo-Turtle and Deployment Tracker callbacks over plain HTTP | Low | Code-discovered | — | + +--- + +## Past Security Incidents + +No known security incidents documented in code, wiki, or Jira at time of generation. + +--- + +<!-- security-init: generated-at=2026-05-12T00:00:00Z base-sha=1495ffb862641ed7220b0cbce5c021e7487f2a1d --> diff --git a/docs/acronyms.md b/docs/acronyms.md new file mode 100644 index 0000000..16df99f --- /dev/null +++ b/docs/acronyms.md @@ -0,0 +1,18 @@ +# Acronyms + +> Domain-specific acronyms used in this repo's code and docs. Generic programming +> acronyms (HTTP, JSON, gRPC, API, etc.) are not listed here. + +| Acronym | Definition | Used in | +|---------|------------|---------| +| BU | Business Unit — partitions services for multi-tenancy; drives Helm chart paths, ArgoCD namespaces, and GCP project naming | infrastructure | +| BUILDKIT | Docker BuildKit — advanced build subsystem; referenced only in the Node.js builder (`buildNode.groovy`) where it is explicitly disabled via `DOCKER_BUILDKIT=0` | flows | +| CAC | Config-as-Code — Meesho's convention for validating application YAML configs (`application-*.yml`) against a schema at PR time; gated by the `ValidateCacConfig` whitelist | flows, policy | +| CICDF | CICD Flow — the main pipeline orchestration function (`commonCICDFlow`) that sequences build, deploy, and notify stages | flows, architecture | +| GCPLBP | GCP Load Balancer Project — the GCP project that hosts the load balancer; used as a distinct project reference in `constructParam.groovy` alongside `GCPP` | infrastructure | +| GCPP | GCP Project — the GCP project identifier for the service's deployment target; computed from BU name as `meesho-<bu>-prd-0622` | infrastructure | +| INFRA | Infrastructure environment — `env.INFRA_ENV` controls which Jenkins pod template (`<INFRA_ENV>-pod.yaml`) is selected for the build agent | infrastructure | +| NPMRC | npm configuration file (`.npmrc`) — bundled in the workspace so private registry credentials are available during Node.js builds | flows | +| SCMH | SCM Host — the Source Control Management host URL; passed to checkout stages to resolve repository clone URLs | flows | +| SCMS | Source Control Management System — generic reference to the SCM provider (GitHub) used in checkout and PR-trigger logic | flows | +| SONAR | SonarQube — static analysis tool; skippable via the `skip-sonar-whitelist.yaml` exception list for Maven `prd` builds | policy | diff --git a/docs/adr/0001-single-shared-library-for-all-services.md b/docs/adr/0001-single-shared-library-for-all-services.md new file mode 100644 index 0000000..72f7663 --- /dev/null +++ b/docs/adr/0001-single-shared-library-for-all-services.md @@ -0,0 +1,46 @@ +# ADR-0001: Single Shared Library Consumed by All Services + +**Status:** Accepted +**Category:** PATTERN +**Date decided:** Project inception +**Date documented:** 2026-05-12 + +## Context + +Meesho runs 100+ microservices across multiple teams. Each service needs CI/CD: build, test, sonar analysis, Docker image push, and ArgoCD deployment. Before devops-lib, teams owned their own Jenkinsfiles and copy-pasted pipeline definitions from each other, resulting in drift, inconsistent policy enforcement, and no central control over who could bypass sonar gates or deploy directly to production. + +## Decision + +All Meesho microservices consume a single Jenkins Shared Library — `devops-lib` — via `@Library('devops-lib@main')` in their Jenkinsfile. The library exposes a single entry point (`eksCICD`) that handles the full CI/CD lifecycle. Service teams do not write pipeline logic; they only provide a `config.yaml`. + +## Alternatives Considered + +- **Per-team Jenkinsfiles**: Rejected because teams were already copy-pasting pipelines and the resulting drift made it impossible to enforce sonar gates, whitelist checks, or deployment policies uniformly. +- **Per-language pipeline templates**: Rejected in favour of a single entry point that dispatches to language-specific stages internally — cross-cutting concerns (sonar, notifications, ArgoCD) remain in one place. + +## Consequences + +**Positive:** +- Policy enforcement (sonar gates, whitelist checks, Ringmaster gating) is applied uniformly to every service on every build — no team can accidentally opt out. +- Infrastructure changes (new deploy strategy, new ArgoCD step, new policy) roll out to all 100+ services by merging one PR to devops-lib. +- New service onboarding is reduced to writing a `config.yaml` — no pipeline expertise needed from the service team. + +**Negative:** +- The library becomes a dependency for every service build — a broken devops-lib main branch blocks all CI/CD. +- Teams needing custom pipeline behaviour have limited escape hatches; they must go through the DevOps Platform team. + +**Neutral:** +- devops-lib must support all build types (Maven, Go, Node, Python, Gradle, PHP, Rust) internally, increasing the library's surface area. + +## Constraints + +Meesho's scale (100+ services) made per-service pipeline ownership operationally unsustainable. The decision was also driven by a need for auditable deployment policy — the DevOps team needed a single control plane for all CI/CD. + +## Current Assessment + +**Still appropriate** — no changes needed. + +## Notes + +- Key files: `vars/eksCICD.groovy` (entry point), `src/com/meesho/stages/buildObjHelper.groovy` (language dispatch) +- Consumer services call the library with a single line: `@Library('devops-lib@main') _` diff --git a/docs/adr/0002-branch-name-as-sole-environment-selector.md b/docs/adr/0002-branch-name-as-sole-environment-selector.md new file mode 100644 index 0000000..35055eb --- /dev/null +++ b/docs/adr/0002-branch-name-as-sole-environment-selector.md @@ -0,0 +1,58 @@ +# ADR-0002: Branch Name as the Sole Environment Selector + +**Status:** Accepted +**Category:** PATTERN +**Date decided:** Project inception +**Date documented:** 2026-05-12 + +## Context + +Each Meesho service needs to deploy to different environments (staging, production, integration, feature). A naive approach would let each service declare its target environment in config, but this creates a risk: a misconfigured service could accidentally deploy to production from a feature branch, or fail to promote through the standard develop → stg → main → prd path. + +## Decision + +`cicd_environment` is derived entirely and deterministically from the branch name. No per-service environment configuration exists. The mapping is: + +| Branch | PR target | `cicd_environment` | +|---|---|---| +| `main` / `master` / `gcp-main` | — | `prd` | +| `develop` | — | `stg` | +| any | `main` | `int` | +| any | `develop` | `ftr` | +| `hotfix/*` | — | `prd` (sonar + tests skipped) | + +## Alternatives Considered + +- **Per-service environment configuration**: Rejected because it creates a class of misconfiguration bugs (wrong env in config.yaml → wrong deploy target) and makes it impossible to enforce the standard branch promotion strategy uniformly. +- **Environment as a Jenkins parameter**: Rejected because it relies on the engineer picking the right environment at trigger time — error-prone and not auditable. + +## Consequences + +**Positive:** +- Services cannot accidentally deploy to prd from a feature branch — the branch IS the environment contract. +- Standard develop → stg → main → prd promotion is enforced uniformly across all services. +- Environment logic lives in one place (`constructParam.groovy`) — easy to audit and change centrally. + +**Negative:** +- Teams with non-standard branch strategies (e.g., release/* branches) cannot use the standard pipeline without DevOps involvement. +- The hotfix path (prd with sonar skipped) requires following the `hotfix/*` naming convention exactly. + +**Neutral:** +- The `int` and `ftr` environments are determined by PR target, not branch name — this is the one case where branch name alone is insufficient. + +## Constraints + +Meesho's deployment policy required a standard promotion path. Allowing per-service environment configuration would have made it impossible to enforce this uniformly across 100+ services. + +## Current Assessment + +**Still appropriate** — no changes needed. + +## Related Decisions + +- [ADR-0001: Single Shared Library](0001-single-shared-library-for-all-services.md) — The shared library enforces this mapping; per-service Jenkinsfiles could override it. + +## Notes + +- Key file: `src/com/meesho/utilities/constructParam.groovy` — environment mapping logic +- PR context is detected via `env.CHANGE_ID` (set by GitHub Branch Source plugin), not by `env.BRANCH_NAME =~ /PR-/` diff --git a/docs/adr/0003-policy-exceptions-in-separate-whitelist-repo.md b/docs/adr/0003-policy-exceptions-in-separate-whitelist-repo.md new file mode 100644 index 0000000..809982d --- /dev/null +++ b/docs/adr/0003-policy-exceptions-in-separate-whitelist-repo.md @@ -0,0 +1,60 @@ +# ADR-0003: Policy Exceptions Controlled by a Separate Meesho/whitelists Repo + +**Status:** Accepted +**Category:** PATTERN +**Date decided:** Early on +**Date documented:** 2026-05-12 + +## Context + +The shared CI/CD library enforces several policies by default: SonarQube quality gate, AppConfig validation, CAC config validation, multizone deployment restrictions. Some services legitimately need to bypass these policies (e.g., a legacy service that cannot pass sonar without significant rework, or a service that doesn't use AppConfig). The question is where to store these exceptions and who can grant them. + +## Decision + +Policy exceptions are controlled by YAML files in a separate `Meesho/whitelists` repository, not by flags in each service's `config.yaml`. There are five active whitelist files: + +| Whitelist | Controls | +|---|---| +| `skip-sonar-whitelist` | Repos that skip SonarQube scanning | +| `app-config-disabled` | Repos that skip AppConfig validation | +| `multizone-enabled-repos` | Repos that must deploy via Ringmaster (not Jenkins) | +| `allowedNonDevelopPrDeploymentToInt` | Repos allowed to deploy non-develop PRs to int | +| `ValidateCacConfig` | Repos that run CAC config validation | + +Adding or removing a repo from any whitelist requires a PR to `Meesho/whitelists` reviewed and merged by the DevOps Platform team. + +## Alternatives Considered + +- **Flags in each service's config.yaml** (e.g., `skip_sonar: true`): Rejected because service teams could self-grant bypass without DevOps review, making it impossible to audit or enforce policy. Note: `config.yaml` does contain a `skip_sonar` field, but `constructParam.groovy` explicitly throws an exception if a repo sets it to `true` without being on the skip-sonar-whitelist — the whitelist is the authoritative gate. +- **Hardcoded exceptions in devops-lib source**: Rejected because adding an exception would require a devops-lib PR, which is heavier than a whitelists PR and conflates policy with pipeline logic. + +## Consequences + +**Positive:** +- DevOps Platform team retains ownership of all policy exception grants — service teams cannot bypass sonar or multizone enforcement unilaterally. +- All exceptions are visible in one repo — easy to audit who has what bypass and why. +- Policy can be tightened or relaxed without touching devops-lib or any service repo. + +**Negative:** +- Adding a legitimate exception requires a separate PR to a different repo, adding friction for service teams. +- The whitelist repo is a single point of dependency — if it's unreachable, every build that checks whitelists fails. + +**Neutral:** +- The whitelist is re-cloned fresh on every build (see ADR-0004), so changes take effect immediately without a devops-lib release. + +## Constraints + +Audit requirements and DevOps policy ownership drove the decision to separate exception management from service config. + +## Current Assessment + +**Still appropriate** — no changes needed. + +## Related Decisions + +- [ADR-0004: Fresh Whitelist Clone Per Build](0004-fresh-whitelist-clone-per-build.md) — How whitelist changes take effect immediately. + +## Notes + +- Key file: `src/com/meesho/utilities/constructParam.groovy:getWhitelistedRepos()` — all whitelist lookups go through this method +- `config.yaml` has a `skip_sonar` field but it is rejected by `constructParam.groovy` unless the repo is also on the whitelist diff --git a/docs/adr/0004-fresh-whitelist-clone-per-build.md b/docs/adr/0004-fresh-whitelist-clone-per-build.md new file mode 100644 index 0000000..4373fa9 --- /dev/null +++ b/docs/adr/0004-fresh-whitelist-clone-per-build.md @@ -0,0 +1,50 @@ +# ADR-0004: Whitelist Cloned Fresh on Every Build — No Caching + +**Status:** Accepted +**Category:** PATTERN +**Date decided:** Early on +**Date documented:** 2026-05-12 + +## Context + +`constructParam.groovy` checks several whitelists (sonar skip, multizone, AppConfig, CAC, non-develop PR deploy) on every build. Jenkins agents are long-lived processes that handle many builds sequentially. If the whitelist were cached in agent memory or on disk, a policy change (adding or removing a repo from a whitelist) would not take effect until the agent restarted or the cache expired. + +## Decision + +`getWhitelistedRepos()` clones `Meesho/whitelists` fresh from GitHub on every build invocation. No in-memory cache, no disk cache, no TTL — each build gets the current live state of the whitelist at that moment. + +## Alternatives Considered + +- **Cache with TTL (e.g., 5 minutes)**: Rejected because a DevOps engineer who merges a critical policy change (adding a repo to skip-sonar during an incident) would have to wait for the cache to expire — unacceptable for incident response. +- **Cache per Jenkins agent restart**: Rejected for the same reason — agents can run for hours/days, making cache invalidation unpredictable. +- **Webhook-triggered cache invalidation**: Not evaluated — the simplicity of a fresh clone was preferred over building an invalidation mechanism. + +## Consequences + +**Positive:** +- Policy changes take effect on the very next build after the whitelist PR is merged — no devops-lib release required. +- No cache invalidation complexity; the whitelist state is always authoritative. + +**Negative:** +- Each build that checks a whitelist incurs a `git clone` of `Meesho/whitelists`. At high build throughput, this is measurable latency. +- If `Meesho/whitelists` is temporarily unreachable (GitHub outage, network partition), all builds that check whitelists fail. + +**Neutral:** +- Multiple whitelist checks in a single build (sonar + AppConfig + multizone) each clone the repo separately — there is no deduplication within a single build. + +## Constraints + +Incident response requirements made immediate policy enforcement non-negotiable. The extra clone latency was accepted as the cost of correctness. + +## Current Assessment + +**Still appropriate** — no changes needed. + +## Related Decisions + +- [ADR-0003: Policy Exceptions in Separate Whitelist Repo](0003-policy-exceptions-in-separate-whitelist-repo.md) — The whitelist repo this decision is about. + +## Notes + +- Key file: `src/com/meesho/utilities/constructParam.groovy:getWhitelistedRepos()` +- **Never refactor this to cache across calls** — the fresh-clone behaviour is load-bearing for incident response. This is documented in CLAUDE.md under NEVER DO. diff --git a/docs/adr/0005-config-only-change-detection-skip-build.md b/docs/adr/0005-config-only-change-detection-skip-build.md new file mode 100644 index 0000000..6c4b6d3 --- /dev/null +++ b/docs/adr/0005-config-only-change-detection-skip-build.md @@ -0,0 +1,46 @@ +# ADR-0005: Config-Only Change Detection — Skip Binary Build, Reuse Latest Image + +**Status:** Accepted +**Category:** PATTERN +**Date decided:** Mid-project +**Date documented:** 2026-05-12 + +## Context + +Meesho services store both application code and deployment configuration (Helm values, AppConfig YAML) in the same repository. Teams frequently push config-only changes — tweaking memory limits, updating feature flags, changing environment variables — that do not require recompiling the binary or rebuilding the Docker image. Without detection, every such commit triggers a full 10–15 minute CI run: compile, test, sonar scan, Docker build, image push — then deploy the same binary that was already running. + +## Decision + +Each build stage checks whether the Git diff contains only `*.yaml` file changes (no source code). If so, the binary build, Docker build, and image push are entirely skipped. The latest image tag is fetched from Google Artifact Registry (GAR) and used directly for the ArgoCD deployment. The full build runs only when source code changes are present. + +## Alternatives Considered + +- **Always run the full build**: Rejected — config rollouts would take 10–15 minutes when the only change is a YAML file, causing friction and delaying incident response (e.g., bumping a memory limit during an OOM incident). +- **Separate repos for code and config**: Considered but rejected — splitting config into a separate repo adds operational complexity (two PRs for one change, out-of-sync risk) without proportional benefit. + +## Consequences + +**Positive:** +- Config rollouts (AppConfig changes, Helm value tweaks, memory limit bumps) complete in ~2 minutes instead of 10–15 minutes. +- Reduces unnecessary Docker image churn — no new SHA for a commit that didn't change the binary. +- Faster incident response: an engineer can bump `memory_request` during an OOM and see it deployed in minutes. + +**Negative:** +- The detection is heuristic — it checks file extensions, not semantic content. A YAML file that configures build behaviour (e.g., a hypothetical `.github/workflows/` file) would be misclassified as a config-only change. +- The latest image tag from GAR must exist; if the previous build failed before pushing an image, a config-only change will fail to find a tag to deploy. + +**Neutral:** +- The config-only path still triggers the ArgoCD deployment steps — only the build and image push are skipped. + +## Constraints + +Build time was the primary constraint. Teams were complaining about slow feedback cycles for config changes. The detection logic was the minimal implementation that addressed this without a repo restructure. + +## Current Assessment + +**Still appropriate** — no changes needed. + +## Notes + +- Key file: `src/com/meesho/stages/buildGo.groovy:is_config_only_change_and_should_deploy_argo()` (reference implementation; similar logic exists in other build stages) +- The TODO comment in `buildMaven.groovy:70` notes that appConfig changes also currently trigger a build — this is a known gap diff --git a/docs/adr/0006-ringmaster-mandatory-build-trigger-gate.md b/docs/adr/0006-ringmaster-mandatory-build-trigger-gate.md new file mode 100644 index 0000000..675b281 --- /dev/null +++ b/docs/adr/0006-ringmaster-mandatory-build-trigger-gate.md @@ -0,0 +1,48 @@ +# ADR-0006: Ringmaster as Mandatory Build Trigger Gate + +**Status:** Accepted +**Category:** PATTERN +**Date decided:** Early on +**Date documented:** 2026-05-12 + +## Context + +Jenkins is accessible to all engineers in the organisation. Without a trigger gate, any engineer can click "Build Now" in Jenkins and kick off a build — including a production deployment — without any audit trail, approval, or callback to downstream systems. Ringmaster is Meesho's internal deployment orchestration system that tracks every deployment: who triggered it, what tag was deployed, when, and whether it succeeded. Turbo-Turtle is the CI callback system that receives the build result and updates deployment state. + +## Decision + +`eksCICD.groovy` hard-rejects any build not triggered by `ringmaster-bot`, `turbo-turtle`, or a hardcoded DevOps allowedUsers list. The rejection is immediate and explicit — the build errors with a message directing the engineer to use Ringmaster instead. The string `"ringmaster-bot"` is the sole signal that routes CI callbacks correctly between Ringmaster and Turbo-Turtle. + +## Alternatives Considered + +- **Allow direct Jenkins triggers with a warning**: Rejected — a warning is easily ignored; the deployment would still bypass Ringmaster's tracking and the Turbo-Turtle callback would have no caller to report to. +- **Restrict Jenkins UI access via RBAC**: Considered but not implemented — coarse-grained Jenkins RBAC would still allow authorized engineers to trigger builds directly, and doesn't solve the callback problem. +- **Audit log only (no rejection)**: Rejected — audit-only does not prevent the problem; it only discovers it after the fact. + +## Consequences + +**Positive:** +- Every production deployment is tracked in Ringmaster — who triggered it, what tag, when — creating a full deployment ledger. +- Turbo-Turtle always receives the CI result callback because the trigger is always one of the known callers. +- Engineers cannot bypass deployment holds or approval flows by triggering Jenkins directly. + +**Negative:** +- DevOps engineers must maintain the hardcoded `allowedUsers` list for emergency access (e.g., debugging a pipeline issue directly from Jenkins). +- The gate adds a hard dependency on Ringmaster being operational for any build to run. + +**Neutral:** +- The string `"ringmaster-bot"` is load-bearing — changing it without coordinating with the Ringmaster team would break the callback routing between Ringmaster and Turbo-Turtle. + +## Constraints + +Meesho's deployment audit and compliance requirements necessitated a full deployment ledger. The Ringmaster/Turbo-Turtle architecture was already in place; the gate was added to enforce its use. + +## Current Assessment + +**Still appropriate** — no changes needed. + +## Notes + +- Key file: `vars/eksCICD.groovy:12-26` — allowedUsers list and trigger validation +- **Never change the string `"ringmaster-bot"`** without coordinating with the Ringmaster team — documented in CLAUDE.md under NEVER DO +- Ringmaster UI: `https://ringmaster.meeshogcp.in/applications/cicd/home` diff --git a/docs/adr/0007-gitops-via-argocd-4-step-sync-sequence.md b/docs/adr/0007-gitops-via-argocd-4-step-sync-sequence.md new file mode 100644 index 0000000..b7007e1 --- /dev/null +++ b/docs/adr/0007-gitops-via-argocd-4-step-sync-sequence.md @@ -0,0 +1,56 @@ +# ADR-0007: GitOps Deployments via Strict 4-Step ArgoCD Sync Sequence + +**Status:** Accepted +**Category:** INFRA +**Date decided:** Early on +**Date documented:** 2026-05-12 + +## Context + +Meesho migrated GCP service deployments from direct `kubectl apply` / Helm install to GitOps via ArgoCD. The key requirement was continuous reconciliation — the cluster state should always reflect what's in Git, and any manual `kubectl` changes should be automatically reverted. The deployment pipeline needed to update two separate Git repositories (argo-config for ArgoCD Application manifests, helm-repo for Helm chart values) and trigger ArgoCD to sync, without leaving the cluster in an inconsistent intermediate state. + +## Decision + +All GCP service deployments go through a strict 4-step sequence in `deployArgoCD.groovy`: + +1. **`update_argo_repo`** — Push the updated ArgoCD Application manifest to devops-argo-config +2. **`refresh_app_of_apps`** — Trigger ArgoCD to sync the app-of-apps, creating any new Application objects +3. **`update_helm_repo`** — Push the new Helm chart (with the new image tag) to the Helm repo +4. **`refresh_and_sync`** — Trigger ArgoCD to sync the specific application + +The order is non-interchangeable. Steps 2 and 4 cannot be swapped. + +## Alternatives Considered + +- **Direct `kubectl apply`**: Rejected — any manual change to the cluster would persist indefinitely; no drift detection or automatic reconciliation. +- **Helm install from Jenkins directly**: Rejected — Helm state would live only in the cluster's release history, not in Git; no GitOps audit trail or rollback via git revert. +- **Skipping step 2 (app-of-apps refresh)**: Not a conscious alternative — the hard requirement emerged from debugging. For new services, step 2 must run before step 3 because the ArgoCD Application object doesn't exist yet; if step 3 (Helm push) runs first, step 4 (sync) targets a non-existent application and fails silently. + +## Consequences + +**Positive:** +- Every deployment is a Git commit — rollback is a git revert, and the cluster state is always reproducible from Git history. +- ArgoCD continuously reconciles cluster state — manual `kubectl apply` changes are automatically reverted, preventing configuration drift. +- Deployment failures are localised: the 4-step sequence makes it clear which step failed (argo-config push? app-of-apps refresh? Helm push? sync?) for faster debugging. + +**Negative:** +- The 4-step sequence is opaque without documentation — engineers debugging a deploy failure must know which step corresponds to which operation. +- Steps 2 and 4 being non-interchangeable is tribal knowledge; swapping them for new services causes a silent sync failure that is hard to diagnose. +- ArgoCD dependency: if ArgoCD is degraded, all deployments are blocked regardless of build success. + +**Neutral:** +- The sequence touches two separate Git repositories (devops-argo-config and helm-repo) in a single pipeline run — partial failures leave one repo updated and the other stale. + +## Constraints + +ArgoCD was the organisational standard for GCP deployments. The 4-step sequence was designed to handle both the case of existing services (steps 1, 3, 4 are the hot path) and new services being onboarded for the first time (step 2 is required to create the Application object before step 4 can sync it). + +## Current Assessment + +**Still appropriate** — no changes needed. + +## Notes + +- Key file: `src/com/meesho/stages/deployArgoCD.groovy` (480+ lines) — all 4 steps are defined here +- The step ordering constraint is documented in CLAUDE.md and `docs/tribal-knowledge.md` (TK#10) as load-bearing tribal knowledge +- See also: `docs/wiki/pages/deploy/argocd-sync.md` for a detailed walkthrough of each step diff --git a/docs/adr/0008-canary-mandatory-for-tier1-services-in-prd.md b/docs/adr/0008-canary-mandatory-for-tier1-services-in-prd.md new file mode 100644 index 0000000..ac7d22d --- /dev/null +++ b/docs/adr/0008-canary-mandatory-for-tier1-services-in-prd.md @@ -0,0 +1,56 @@ +# ADR-0008: Canary Deploy Mandatory for Tier-1 (sp0/up0) Services in Production + +**Status:** Accepted +**Category:** RELIABILITY +**Date decided:** Mid-project +**Date documented:** 2026-05-12 + +## Context + +Meesho's services are classified by priority tier (`sp0`, `up0`, `sp1`, `up1`, etc.). Tier-1 services (`sp0`/`up0`) handle the highest traffic volumes and are critical to core business flows. A bad deploy that hits 100% of production traffic on a Tier-1 service has a catastrophic blast radius — full outage, revenue impact, and customer-facing failure. Teams were inconsistently configuring canary rollouts: some enabled them, some skipped them, and some configured them with `skipAnalysis: true` which bypasses the automated rollout analysis. + +This inconsistency was the contributing factor in at least one production incident where a bad deploy on a Tier-1 service reached full traffic before the issue was detected. + +## Decision + +`deployArgoCD.groovy` enforces canary deployment as a hard requirement for all services with `priority_v2: sp0` or `priority_v2: up0` deploying to the `prd` environment. The enforcement checks: +1. `canary.enabled: true` must be set +2. `canary.skipAnalysis: false` — analysis cannot be bypassed +3. `canary.enableManualPromotion: true` — a human must promote the canary to full traffic + +If any of these conditions are not met, the deployment is blocked with an explicit error: `"Enable canary and retry"`. Enforcement is applied at the library level — service teams cannot override it. + +## Alternatives Considered + +- **Documentation and guidelines only**: Rejected — teams were already aware of canary best practices but inconsistently applied them; a documented recommendation had failed to produce uniform behaviour. +- **Enforcement in Ringmaster only**: Considered but rejected — enforcement at the library level means it applies to all deploy paths, including any future tooling that calls `deployArgoCD.groovy`. +- **Enforce for all services, not just sp0/up0**: Considered but rejected as too disruptive — lower-priority services have smaller blast radii and the overhead of canary analysis was not justified for all tiers. + +## Consequences + +**Positive:** +- Tier-1 bad deploys cannot reach 100% of production traffic without a human promotion step. +- Canary analysis (metrics, error rate) runs automatically before promotion, catching regressions before they impact all users. +- Enforcement is consistent across all Tier-1 services — no team can skip it. + +**Negative:** +- Canary rollouts add time to Tier-1 deployments — promotion requires human action, which can delay hotfixes. +- The hotfix path (`hotfix/*` branches) sets `skipAnalysis: true` to allow bypassing canary analysis in emergencies, which re-introduces the risk for the hotfix scenario. +- Services that newly cross the sp0/up0 threshold must configure canary before their next prd deploy or they will be blocked. + +**Neutral:** +- The enforcement only applies to non-cron, non-worker, non-scheduler, non-consumer deployments — background jobs are excluded. + +## Constraints + +A production incident on a Tier-1 service drove this decision. Post-incident, the risk of leaving canary configuration to team discretion was deemed unacceptable. The library-level enforcement was the fastest way to guarantee coverage across all affected services without requiring each team to update their configuration proactively. + +## Current Assessment + +**Still appropriate** — no changes needed. + +## Notes + +- Key file: `src/com/meesho/stages/deployArgoCD.groovy:407-429` — canary enforcement block +- Hotfix bypass: `value_binding1['canary']['skipAnalysis'] = (env.hot_fix) ? true : ...` at line 355 — hotfixes can bypass canary analysis +- The `addHeadless` flag and Node services are also excluded from enforcement diff --git a/docs/adr/0009-jvm-heap-auto-derived-from-pod-memory-request.md b/docs/adr/0009-jvm-heap-auto-derived-from-pod-memory-request.md new file mode 100644 index 0000000..53a9ddf --- /dev/null +++ b/docs/adr/0009-jvm-heap-auto-derived-from-pod-memory-request.md @@ -0,0 +1,53 @@ +# ADR-0009: JVM Heap Auto-Derived from Pod memory_request + +**Status:** Accepted +**Category:** RELIABILITY +**Date decided:** Mid-project +**Date documented:** 2026-05-12 + +## Context + +Java services running in Kubernetes pods are subject to two memory limits: the pod's `memory_limit` (enforced by the kubelet — exceed it and the pod is OOM-killed) and the JVM's heap size (`-Xmx`). By default, the JVM sets heap to 1/4 of the physical RAM it detects — but inside a container, it detects the node's physical RAM, not the pod's memory limit. A Java service in a pod with `memory_limit: 2Gi` running on a 64Gi node would default to a 16Gi heap, far exceeding its limit and triggering immediate OOM kill. + +Meesho had multiple incidents where Java services were OOM-killed because: +1. The JVM was using the wrong default (node RAM, not pod limit) +2. Teams were setting `-Xmx` manually but forgetting to update it when `memory_request` changed +3. Teams were setting `-Xmx` too high, causing heap to exceed the pod limit + +## Decision + +`deployArgoCD.groovy` automatically computes `xms` and `xmx` from the service's `memory_request` value in `deployment.yaml`. The derived values are injected into the Helm chart at deploy time. Services do not need to set `-Xmx` in `JAVA_OPTS` manually. The escape hatch `jvm_memory_override: true` in `deployment.yaml` allows a service to opt out and set its own JVM flags. + +## Alternatives Considered + +- **Require teams to set -Xmx manually**: Rejected — teams consistently forgot to update `-Xmx` when changing `memory_request`, causing OOM kills after pod resource changes. This happened across multiple services. +- **JVM container awareness flag (-XX:+UseContainerSupport)**: This flag (available in JDK 11+) allows the JVM to read the cgroup limit instead of physical RAM. Not adopted as the primary solution because it requires all services to use JDK 11+ and the flag needs to be explicitly set in each service's startup config — still a per-service manual step. +- **Fixed default heap values**: Rejected — services have wildly different memory requirements; a fixed default would be wrong for most. + +## Consequences + +**Positive:** +- Java services cannot be OOM-killed due to JVM heap misconfiguration — the heap is always proportional to the pod's actual memory allocation. +- Teams never need to update `-Xmx` manually when changing `memory_request` — the library keeps them in sync automatically. +- Eliminates a whole class of incident: "service OOM-killed because someone bumped memory_request but forgot to update -Xmx." + +**Negative:** +- The auto-derived heap may not be optimal for services with unusual heap vs non-heap memory ratios (e.g., services with large off-heap caches). These services must use `jvm_memory_override: true`. +- The calculation logic is not immediately transparent to service teams — they may not know why their `-Xmx` is what it is. + +**Neutral:** +- If `deployment_args` already contains an `-Xmx` or `-Xms` flag, the auto-derive reads and preserves those values rather than overwriting them. `jvm_memory_override` is the clean opt-out for services that need full control. + +## Constraints + +Repeated OOM incidents across multiple Java services drove this decision. The library-level fix was preferred over per-service remediation because the root cause was systemic (wrong JVM defaults in containers) and would recur as long as teams configured heap manually. + +## Current Assessment + +**Still appropriate** — no changes needed. + +## Notes + +- Key file: `src/com/meesho/stages/deployArgoCD.groovy:205-268` — `xms`/`xmx` calculation +- Opt-out: set `jvm_memory_override: true` in `deployment.yaml` to manage JVM flags manually +- Documented in CLAUDE.md: "JVM memory: `deployArgoCD.groovy` auto-calculates `xmx`/`xms` from pod `memory_request`." diff --git a/docs/adr/0010-cloud-and-branch-namespaced-artifact-paths.md b/docs/adr/0010-cloud-and-branch-namespaced-artifact-paths.md new file mode 100644 index 0000000..a0c8d31 --- /dev/null +++ b/docs/adr/0010-cloud-and-branch-namespaced-artifact-paths.md @@ -0,0 +1,59 @@ +# ADR-0010: Cloud-and-branch namespaced artifact paths + +**Status:** Accepted +**Category:** DATA +**Date decided:** Project inception +**Date documented:** 2026-05-13 + +## Context + +Every service build in `devops-lib` produces a deployable artifact (Maven JAR / language-equivalent) plus a Docker image; the pipeline must decide where to store these and whether a re-run on the same commit can short-circuit a rebuild. The library was written when both AWS and GCP backends were on the table, so the storage layer was parameterised by provider scheme; in practice the platform has since standardised on GCP and the `s3://` code paths are vestigial. PRs and release branches share the same Jenkins jobs but have very different trust levels — a PR artifact must never be promotable to a production tag. + +## Decision + +Artifacts are stored under fully-namespaced paths of the form `<scheme>://<bucket>/<repo_name>/<branch_name>/<TAG>/`, where `<scheme>` is `gs://` in active use (`s3://` branches remain in the codebase but are not executed today). Release branches (`main` / `master` / `gcp-main`) and `develop` reuse cached artifacts on re-run via `checkS3()`; PR builds intentionally bypass that check and force a fresh build every time. + +## Alternatives Considered + +No alternatives were explicitly evaluated by the team during this interview. The branch-namespaced layout was the day-one design and has not been revisited. + +## Consequences + +**Positive:** +- A PR's image cannot be confused with a release image at the bucket-path level — provides a structural guarantee against accidental promotion. +- Re-running a build on the same release branch is free (artifact reuse), keeping incremental commits cheap. +- Multi-cloud scheme prefix is harmless even when only one cloud is active; switching back would be a config change, not a rewrite. + +**Negative:** +- `s3://` code paths sit unused in `buildMaven.groovy` and friends — invisible tech debt that confuses new readers and grows the surface area for stale-config bugs. +- The `(branch_name, TAG)` key means renaming a branch or rebasing a PR can leak artifacts into the wrong namespace if `branch_name` is computed loosely. + +**Neutral:** +- Cache reuse is implicit (artifact-exists ⇒ skip build) rather than declared — see ADR-0005 for the config-only fast-path that uses the same mechanism. + +## Constraints + +- Branch identity is the cache key, so `branch_name` must be a stable string for the lifetime of a build chain. PR-target detection (`env.CHANGE_ID`) is load-bearing here. +- AWS code paths exist for historical reasons; today's platform is GCP-only and the team has not undertaken a cleanup pass. + +## Current Assessment + +- **Adequate with caveats** — the strategy is sound; the vestigial AWS branches are noise that should be removed in a separate cleanup. + +## Related Decisions + +- [ADR-0005: Config-Only Change Detection — Skip Binary Build, Reuse Latest Image](0005-config-only-change-detection-skip-build.md) — the same artifact-existence check powers the config-only fast-path. + +## Notes + +- Key files: `src/com/meesho/stages/buildMaven.groovy` +- The `s3://` branches in `buildMaven.groovy` are dead code in current production — flag for cleanup, not for documentation as an alternative. +- Discovery id: DATA-1 + +<!-- adr-generator-meta +discovery_id: DATA-1 +run_id: 90b9a400-266c-4786-b793-d811efa99276 +last_completed_at: 2026-05-13T12:10:00Z +mode: create +cache_uri: gs://ai-blitz-agent-readability/adr/devops-lib/adr-discovery.json +--> diff --git a/docs/adr/0011-build-user-identity-routes-post-build-callbacks.md b/docs/adr/0011-build-user-identity-routes-post-build-callbacks.md new file mode 100644 index 0000000..71eab99 --- /dev/null +++ b/docs/adr/0011-build-user-identity-routes-post-build-callbacks.md @@ -0,0 +1,66 @@ +# ADR-0011: Build-user identity routes post-build callbacks + +**Status:** Accepted +**Category:** COMMUNICATION +**Date decided:** Early on +**Date documented:** 2026-05-13 + +## Context + +Once a build completes, the pipeline must (a) notify the service team, and (b) tell the deployment-orchestration layer that a new image is ready to roll out. Multiple orchestrators consume this signal — Ringmaster and Turbo-Turtle (the actively maintained CD systems), the legacy Deployment Tracker (still backing some downstream tooling), and a toolchain-environment build-callback service used by Node toolchain builds. Each consumer has a different API contract, a different Slack message shape, and a different policy on whether the team channel should be notified at all. The shared library is invoked by both bot-driven CD (Ringmaster / Turbo-Turtle trigger builds via dedicated bot users) and human-driven Jenkins runs (manual deploys, hotfixes, retries by anyone with allowedUsers), so there is no single contract that fits all callers. + +## Decision + +`notify.groovy` routes by the Jenkins `build_user`, which is set by whoever triggered the build: + +- `build_user == "ringmaster-bot"` or `"turbo-turtle"` → call `deployRingmaster.run()` and emit a Ringmaster-flavored Slack message (production gets a richer message with the deploy URL); the generic team notification is suppressed. +- Any other user (manual deploy, hotfix, retry) → fall through to a generic `slackSend` on the team's notify channel. +- `INFRA_ENV == 'toolchain'` → take a completely separate path that POSTs to the toolchain build-callback service and returns early, bypassing the build-user routing entirely. + +In parallel, for release branches (`gcp-main` / `main` / `gcp-master` / `master` / `farmiso-main`), the legacy Deployment Tracker (`postTrackingApi`) is always called, and the Ringmaster history-DB (`postTrackingRingmasterApi`) is called when `env.SERVICES` is set. + +## Alternatives Considered + +No alternatives were explicitly evaluated during this interview. Routing on `build_user` was chosen because it is the only signal available without coordinating an extra config flag with every consumer service. + +## Consequences + +**Positive:** +- New consumers can plug in without re-wiring every consumer service's `config.yaml` — they only need to claim a dedicated bot identity. +- The legacy Deployment Tracker path stays intact for backward compatibility while new traffic flows through Ringmaster. +- Manual / human-triggered builds get the generic Slack notification path so engineers always see a team-channel message regardless of CD orchestrator. + +**Negative:** +- The routing is implicit and not documented in `config.yaml` — a reader of a service's pipeline cannot tell which CD orchestrator will get the callback without grepping `notify.groovy`. +- Renaming or replacing either bot identity is breaking: the strings `"ringmaster-bot"` and `"turbo-turtle"` are hard-coded compare targets (the existing `NEVER DO` list in `CLAUDE.md` calls this out for `ringmaster-bot`). + +**Neutral:** +- The `INFRA_ENV == 'toolchain'` branch sits outside the build-user routing — it is a parallel routing axis (build environment, not trigger identity). + +## Constraints + +- Caller identity is the only signal available at notify time — no config flag is in scope. +- Strict string comparison on bot usernames couples this code to Ringmaster / Turbo-Turtle naming. + +## Current Assessment + +- **Adequate with caveats** — the routing works and survives new CD orchestrators being added, but the dispatch should ideally be table-driven rather than chained `if`s, and the bot-name strings should be configurable rather than hard-coded. + +## Related Decisions + +- [ADR-0006: Ringmaster as Mandatory Build Trigger Gate](0006-ringmaster-mandatory-build-trigger-gate.md) — Ringmaster also gates the *trigger* side of builds; this ADR covers the *callback* side. +- [ADR-0007: GitOps Deployments via Strict 4-Step ArgoCD Sync Sequence](0007-gitops-via-argocd-4-step-sync-sequence.md) — the deploy mechanism that Ringmaster invokes downstream. + +## Notes + +- Key files: `src/com/meesho/stages/notify.groovy`, `src/com/meesho/stages/deployRingmaster.groovy` +- The release-branch dual-write (`postTrackingApi` + `postTrackingRingmasterApi`) coexists with the build-user routing but is orthogonal to it (gated on `BRANCH_NAME` and `env.SERVICES`). +- Discovery id: COMMUNICATION-1 + +<!-- adr-generator-meta +discovery_id: COMMUNICATION-1 +run_id: 90b9a400-266c-4786-b793-d811efa99276 +last_completed_at: 2026-05-13T12:15:00Z +mode: create +cache_uri: gs://ai-blitz-agent-readability/adr/devops-lib/adr-discovery.json +--> diff --git a/docs/adr/0012-string-interpolated-helm-values-from-user-config.md b/docs/adr/0012-string-interpolated-helm-values-from-user-config.md new file mode 100644 index 0000000..d0dbf4d --- /dev/null +++ b/docs/adr/0012-string-interpolated-helm-values-from-user-config.md @@ -0,0 +1,61 @@ +# ADR-0012: String-interpolated Helm values from user config.yaml + +**Status:** Accepted +**Category:** COMMUNICATION +**Date decided:** Project inception +**Date documented:** 2026-05-13 + +## Context + +ArgoCD deploys are driven by Helm charts whose `values.yaml` files are produced per-build by `deployArgoCD.groovy`. The inputs are a service's `deployment.yaml` plus a handful of pipeline-derived fields (image tag, pod resources, JVM heap, etc.). Helm chart `values.yaml` is itself a templating surface — Helm's own `{{ .Values.x }}` syntax reads these files at install time — so anything produced here is interpreted as a template by the next layer, not as plain data. + +## Decision + +`deployArgoCD.groovy` builds the rendered `values.yaml` by string-substituting fields into a stub template via chained `.replaceAll()` calls, rather than constructing it via a YAML library or a typed DTO and re-serialising. The team treats Helm's chart-values surface as a template-on-template stack: re-serialising via a YAML library would re-introduce template-syntax escaping problems (quoting `{{ }}`, preserving multi-line string semantics, handling Helm-specific structural tags) — keeping the entire pipeline string-native is the simpler invariant. + +## Alternatives Considered + +No alternatives were explicitly evaluated by the team during this interview. The "use a typed YAML library / DTO and re-serialise" approach was acknowledged as the obvious counter-proposal but was rejected on the template-on-template grounds above. + +## Consequences + +**Positive:** +- The renderer stays a one-layer string substitution — easy to read, easy to debug from a Jenkins console log. +- No risk of a typed re-serialisation silently re-escaping Helm `{{ }}` templates or stripping comments. +- Matches the rest of the pipeline, which is string-and-`sh` heavy. + +**Negative:** +- User-supplied config values are not validated against a schema before substitution — a service's `deployment.yaml` containing unescaped quotes, colons, or newlines can produce a malformed `values.yaml` (review-learnings PR #343 flagged this). +- The implicit YAML-injection risk depends on every consumer service writing well-formed `deployment.yaml` — there is no guardrail in the library itself. +- Adding a new field requires editing the template stub AND the substitution chain in `deployArgoCD.groovy` together — easy to drift. + +**Neutral:** +- The decision lives entirely in `deployArgoCD.groovy`; switching strategies in future would be local to that file. + +## Constraints + +- Helm chart values are interpreted as templates downstream — any solution must preserve template literals without escaping them. +- The shared library runs in the Jenkins sandbox, which constrains which Java / Groovy serialisation APIs are safely callable. + +## Current Assessment + +- **Adequate with caveats** — the strategy is defensible, but the lack of input-validation guardrails (the YAML-injection surface called out in review learnings) remains an open risk. A targeted schema-validation pass before substitution would mitigate it without changing the rendering strategy. + +## Related Decisions + +- [ADR-0007: GitOps Deployments via Strict 4-Step ArgoCD Sync Sequence](0007-gitops-via-argocd-4-step-sync-sequence.md) — the deploy mechanism that consumes the rendered `values.yaml`. +- [ADR-0009: JVM Heap Auto-Derived from Pod memory_request](0009-jvm-heap-auto-derived-from-pod-memory-request.md) — another pipeline-derived input to the same `values.yaml`. + +## Notes + +- Key files: `src/com/meesho/stages/deployArgoCD.groovy` +- Open risk: review-learnings PR #343 flagged the YAML-injection surface — input validation is the recommended mitigation. +- Discovery id: COMMUNICATION-2 + +<!-- adr-generator-meta +discovery_id: COMMUNICATION-2 +run_id: 90b9a400-266c-4786-b793-d811efa99276 +last_completed_at: 2026-05-13T12:18:00Z +mode: create +cache_uri: gs://ai-blitz-agent-readability/adr/devops-lib/adr-discovery.json +--> diff --git a/docs/adr/0013-multi-zone-deployables-gated-out-of-direct-jenkins-argocd.md b/docs/adr/0013-multi-zone-deployables-gated-out-of-direct-jenkins-argocd.md new file mode 100644 index 0000000..3eb2627 --- /dev/null +++ b/docs/adr/0013-multi-zone-deployables-gated-out-of-direct-jenkins-argocd.md @@ -0,0 +1,62 @@ +# ADR-0013: Multi-zone deployables gated out of direct Jenkins ArgoCD + +**Status:** Accepted +**Category:** RELIABILITY +**Date decided:** Alongside the multi-zone initiative +**Date documented:** 2026-05-13 + +## Context + +Meesho ran a multi-zone initiative to deploy production services across two GCP zones in parallel and split traffic between them. The initiative immediately exposed that not every workload type is safe to multiply across zones: schedulers, cron jobs, and consumers cause correctness issues when more than one zone runs them at the same time; `cache` and `database` service types are explicitly blocked by Turbo-Turtle's `ValidateDeploymentConfigActivity`; some workloads (e.g. Deepgram, with volume affinity that pins it to a single zone) can't go multi-zone at all. Meanwhile, `deployArgoCD.groovy` in this shared library only knows how to call `argocd sync` against a single application — it has no concept of split-by-service-type or per-zone sequencing. + +## Decision + +`deployArgoCD.groovy` fetches the `multizone-enabled-repos` whitelist from `Meesho/whitelists` at the start of every build. If the current deployable is on the list, the pipeline aborts with `"Multi-zone enabled for this deployable. Please use Ringmaster for deployment."` rather than attempting a sync. Multi-zone-enabled services are required to go through Ringmaster, which owns the split-deploy logic, per-service-type validation, and zone-affinity awareness. + +## Alternatives Considered + +No alternatives were explicitly evaluated by the team during this interview. The gate was introduced alongside the multi-zone initiative itself, not retrofitted after an incident. + +## Consequences + +**Positive:** +- Direct Jenkins ArgoCD is structurally incapable of getting multi-zone wrong because it never gets the chance — the gate fails closed. +- The split-deploy logic and the service-type-validation logic live in exactly one place (Ringmaster); we do not maintain two copies. +- A team can flip a service to multi-zone by adding it to `multizone-enabled-repos.yaml` without modifying `devops-lib` — the policy change takes effect on the next build. + +**Negative:** +- Engineers who routinely use `cicd-` Jenkins jobs see an unfamiliar refusal once their service is added to the whitelist; the error message is the only signal pointing them at Ringmaster. +- The library has no way to attempt a partial deploy or to surface what would-have-been-deployed; the gate is binary. + +**Neutral:** +- The decision lives across two repos — this code in `devops-lib`, the source-of-truth list in `Meesho/whitelists`. See ADR-0003 for the broader policy-in-a-separate-repo pattern. + +## Constraints + +- Multi-zone correctness requires service-type awareness that the Jenkins ArgoCD path does not have. +- The whitelist is fetched fresh per build (ADR-0004), so policy changes in `Meesho/whitelists` take effect on the next build with no library release. + +## Current Assessment + +- **Still appropriate** — fail-closed gating is the right posture for a capability the library cannot safely implement. + +## Related Decisions + +- [ADR-0003: Policy Exceptions Controlled by a Separate Meesho/whitelists Repo](0003-policy-exceptions-in-separate-whitelist-repo.md) — explains the whitelist-source-of-truth pattern this gate uses. +- [ADR-0004: Whitelist Cloned Fresh on Every Build — No Caching](0004-fresh-whitelist-clone-per-build.md) — why this list takes effect immediately. +- [ADR-0006: Ringmaster as Mandatory Build Trigger Gate](0006-ringmaster-mandatory-build-trigger-gate.md) — Ringmaster's parallel role on the trigger side. +- [ADR-0011: Build-user identity routes post-build callbacks](0011-build-user-identity-routes-post-build-callbacks.md) — the callback flow that Ringmaster uses on success. + +## Notes + +- Key files: `src/com/meesho/stages/deployArgoCD.groovy` +- Whitelist source: `https://github.com/Meesho/whitelists/blob/main/multizone-enabled-repos.yaml` +- Discovery id: RELIABILITY-1 + +<!-- adr-generator-meta +discovery_id: RELIABILITY-1 +run_id: 90b9a400-266c-4786-b793-d811efa99276 +last_completed_at: 2026-05-13T12:21:00Z +mode: create +cache_uri: gs://ai-blitz-agent-readability/adr/devops-lib/adr-discovery.json +--> diff --git a/docs/adr/0014-open-dependabot-critical-alerts-block-builds.md b/docs/adr/0014-open-dependabot-critical-alerts-block-builds.md new file mode 100644 index 0000000..4122015 --- /dev/null +++ b/docs/adr/0014-open-dependabot-critical-alerts-block-builds.md @@ -0,0 +1,58 @@ +# ADR-0014: Open Dependabot CRITICAL alerts block builds + +**Status:** Accepted +**Category:** RELIABILITY +**Date decided:** Mid-project +**Date documented:** 2026-05-13 + +## Context + +Every Meesho service ships its source dependencies as part of its container image. GitHub's Dependabot continuously scans those dependency manifests against the GitHub Advisory Database and surfaces alerts by severity. Without an enforcement teeth in the pipeline, CRITICAL CVEs can sit open for arbitrary time — merge gates can be bypassed (admin merges, hotfixes, repos that don't enforce branch protection), and a periodic audit only tells security org-wide rather than putting accountability on the team that's actively trying to ship. + +## Decision + +`deployArgoCD.groovy:dependabotCriticalCheck()` calls the GitHub Dependabot Alerts API for the repo on every build. If the API returns any open alert with severity `CRITICAL`, the pipeline aborts with `"Critical vulnerabilities found in repo: <name>. Please resolve the alerts marked with CRITICAL here and retry: <github dependabot URL>"`. There is no whitelist, no override flag, and no manual bypass available from the pipeline side. + +## Alternatives Considered + +N/A — organizational mandate. The Meesho security org required hard enforcement of the CRITICAL-CVE SLA on shipping code; the build-time gate is the implementation choice that satisfies it. + +## Consequences + +**Positive:** +- The deploying team is forced to act on the CVE (fix, escalate, or coordinate with the dependency owner) before they can ship — accountability lands on whoever is actively trying to push, not on a central security team. +- Confirmed firing in practice on real services (Farmiso-Backend, supplier_platform_insights, meesho-web-reels, supplier_platform_payouts as seen in #devops-tech) — the gate is not theoretical. + +**Negative:** +- Engineers blocked by a CRITICAL alert mid-deploy have no in-pipeline bypass — even for hotfixes that are unrelated to the vulnerable dependency. +- The check depends on GitHub Dependabot's classification; a false-positive at CRITICAL would force a real outage detour. + +**Neutral:** +- The check is per-build, not per-PR, so a recently-disclosed CVE can suddenly start blocking deploys for repos that have not changed. + +## Constraints + +- Meesho security org policy: open CRITICAL Dependabot alerts must be resolved before code is shipped. The build-time gate is the enforcement surface for that policy. +- Dependabot must remain enabled on every repo for the gate to be meaningful; that side of the policy lives in repo settings, not in this library. + +## Current Assessment + +- **Still appropriate** — the mandate stands and the gate is the correct enforcement point. + +## Related Decisions + +- [ADR-0006: Ringmaster as Mandatory Build Trigger Gate](0006-ringmaster-mandatory-build-trigger-gate.md) — another build-time hard gate; same architectural pattern of "fail closed at the pipeline boundary." + +## Notes + +- Key files: `src/com/meesho/stages/deployArgoCD.groovy` +- The check is unconditional — there is no whitelist of repos exempted from this gate. +- Discovery id: RELIABILITY-2 + +<!-- adr-generator-meta +discovery_id: RELIABILITY-2 +run_id: 90b9a400-266c-4786-b793-d811efa99276 +last_completed_at: 2026-05-13T12:23:00Z +mode: create +cache_uri: gs://ai-blitz-agent-readability/adr/devops-lib/adr-discovery.json +--> diff --git a/docs/adr/0015-node-install-logic-paired-across-buildnode-groovy-and-dockerfile.md b/docs/adr/0015-node-install-logic-paired-across-buildnode-groovy-and-dockerfile.md new file mode 100644 index 0000000..33c0d47 --- /dev/null +++ b/docs/adr/0015-node-install-logic-paired-across-buildnode-groovy-and-dockerfile.md @@ -0,0 +1,59 @@ +# ADR-0015: Node install logic paired across buildNode.groovy and Dockerfile + +**Status:** Accepted +**Category:** PATTERN +**Date decided:** Early on +**Date documented:** 2026-05-13 + +## Context + +Meesho Node services use both `npm` and `pnpm` depending on the repo. Some teams need to pass extra flags (e.g. `--legacy-peer-deps`) and the shared library cannot anticipate every combination. The install step has to run inside the Docker build (the image needs `node_modules` baked in), but the choice of *which* package manager and *which* flags to run is information the Dockerfile alone can't recover from a clean container — it has to be told. + +## Decision + +The install step is split deliberately across the two files. `src/com/meesho/stages/buildNode.groovy` does the detection: it reads `config.yaml`, sees whether the repo uses `npm` or `pnpm`, and resolves the optional `npm_install_arg` override; it then passes the resolved values to `resources/com/meesho/node-Dockerfile` as Docker build-args. The Dockerfile does the execution: it consumes those build-args and runs the actual install, falling back to a sane default in an `else` branch when no value was passed. Detection lives at the layer that can see the config; execution lives at the layer that runs inside the container. + +## Alternatives Considered + +No alternatives were explicitly evaluated by the team during this interview. The split mirrors what is naturally separable — config interpretation vs runtime install — and has not been revisited. + +## Consequences + +**Positive:** +- Teams can override the install command for their service through `npm_install_arg` in `config.yaml` without forking the library Dockerfile — observed in #devops-tech (Nov 2025) when a team added `--legacy-peer-deps` purely via config. +- The Dockerfile's `else` fallback means a developer can `docker build` locally without Jenkins or `config.yaml` and still get a working install. +- Neither layer needs to know more than it actually does: Groovy doesn't run `npm`, the Dockerfile doesn't read YAML. + +**Negative:** +- Any change to install behaviour has to land in both files at once (tribal-knowledge #11). The pair-edit invariant is real but has not yet caused a production incident. +- Build-args are stringly-typed, so a typo on either side fails late — at install time inside the Docker build. + +**Neutral:** +- The "detection in Groovy, execution in Dockerfile" split is mirrored elsewhere in `devops-lib` (e.g. Maven, Go); Node just happens to be the most config-driven and therefore the most visible example. + +## Constraints + +- The package-manager choice has to be visible at Docker-build time inside the container; build-args are the cleanest way to inject it. +- The library must support repos that build locally (no Jenkins, no `config.yaml`) for developer ergonomics. + +## Current Assessment + +- **Still appropriate** — the split works, the override knob is being used as intended, and no incident has surfaced. A future hardening could add a CI check that any PR touching the install logic on one side also touches the other, but it is not load-bearing today. + +## Related Decisions + +None directly. The "detection in Groovy, execution in container" split is a pattern repeated in other build stages (Maven, Go), but each has its own per-language ADR scope. + +## Notes + +- Key files: `src/com/meesho/stages/buildNode.groovy`, `resources/com/meesho/node-Dockerfile` +- `npm_install_arg` in `config.yaml` is a documented public extension point — see service-team usage in #devops-tech (Nov 2025) for the `--legacy-peer-deps` case. +- Discovery id: PATTERN-2 + +<!-- adr-generator-meta +discovery_id: PATTERN-2 +run_id: 90b9a400-266c-4786-b793-d811efa99276 +last_completed_at: 2026-05-13T12:28:00Z +mode: create +cache_uri: gs://ai-blitz-agent-readability/adr/devops-lib/adr-discovery.json +--> diff --git a/docs/adr/0016-per-environment-helm-chart-versioning.md b/docs/adr/0016-per-environment-helm-chart-versioning.md new file mode 100644 index 0000000..5160c49 --- /dev/null +++ b/docs/adr/0016-per-environment-helm-chart-versioning.md @@ -0,0 +1,60 @@ +# ADR-0016: Per-environment Helm chart versioning (values_v2 vs values_v3) + +**Status:** Accepted +**Category:** INFRA +**Date decided:** During the multi-zone initiative +**Date documented:** 2026-05-13 + +## Context + +Production was the first (and so far only) environment where Meesho rolled out multi-zone deployments — running services in two GCP zones in parallel with split-deploy and per-service-type validation (see ADR-0013). Multi-zone awareness required a new shape of Helm chart values: per-zone overlay files (`gcp-ase1a-values.yaml`, etc.), service-type metadata, and additional tuning. Lower environments (`stg`, `int`, `ftr`) do not run multi-zone today and have no plan to. Forcing them onto the multi-zone-shaped chart would either require multi-zone setup they don't have, or carry chart fields they never use. + +## Decision + +`constructParam.groovy` picks the Helm chart values path per environment: `helmChartsPath = (env == 'prd') ? 'values_v3' : 'values_v2'`. Production reads from `values_v3/` in `devops-helm-charts` (multi-zone-aware); every other environment reads from `values_v2/` (single-zone). Both versions are actively maintained — `values_v2` is the live chart for non-prod, not a deprecated legacy path. + +## Alternatives Considered + +No alternatives were explicitly evaluated by the team during this interview. A unified chart that handles both single-zone and multi-zone via conditional logic was the obvious counter-proposal but was not taken; the team kept the two versions side-by-side instead. + +## Consequences + +**Positive:** +- Non-prod chart stays simple — no multi-zone-only fields polluting the values tree for engineers and reviewers who don't need them. +- Multi-zone schema can evolve in `values_v3` without coordinating breaking changes against non-prod chart consumers. +- Production deploys exercise a chart shape that matches production's runtime topology exactly; no "unused config" surface. + +**Negative:** +- A service that exists in prd has its values defined in two places (`values_v2/<bu>/.../values.yaml` and `values_v3/<bu>/.../values.yaml`); a config change relevant to both has to land in both, and drift between them is silent. +- Engineers debugging an issue have to know which version their environment uses before they can find the right file. + +**Neutral:** +- The decision lives in one line of `constructParam.groovy`; flipping a new env onto `values_v3` is trivial when its multi-zone story is ready. + +## Constraints + +- Multi-zone requires per-zone overlay files and service-type validation that `values_v2` does not have. +- Non-prod environments do not run multi-zone and have no roadmap to; the v3 shape would carry dead configuration there. + +## Current Assessment + +- **Adequate with caveats** — splitting by environment maps cleanly to the runtime topology, but the dual maintenance burden grows linearly with active services and is invisible to anyone not in the DevOps loop. Worth revisiting if/when non-prod gains multi-zone. + +## Related Decisions + +- [ADR-0013: Multi-zone deployables gated out of direct Jenkins ArgoCD](0013-multi-zone-deployables-gated-out-of-direct-jenkins-argocd.md) — the multi-zone initiative that drove the `values_v3` rollout. +- [ADR-0007: GitOps Deployments via Strict 4-Step ArgoCD Sync Sequence](0007-gitops-via-argocd-4-step-sync-sequence.md) — the deploy mechanism that consumes whichever chart values path is selected here. + +## Notes + +- Key files: `src/com/meesho/utilities/constructParam.groovy` +- Chart source: `https://github.com/Meesho/devops-helm-charts` — `values_v2/` and `values_v3/` are sibling top-level directories. +- Discovery id: INFRA-1 + +<!-- adr-generator-meta +discovery_id: INFRA-1 +run_id: 90b9a400-266c-4786-b793-d811efa99276 +last_completed_at: 2026-05-13T12:31:00Z +mode: create +cache_uri: gs://ai-blitz-agent-readability/adr/devops-lib/adr-discovery.json +--> diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..3b71f75 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,43 @@ +# Architecture Decision Records + +This directory contains Architecture Decision Records (ADRs) for devops-lib. +These were retroactively documented on 2026-05-12 through codebase analysis and developer interviews. + +## What is an ADR? + +An Architecture Decision Record captures an important architectural decision along with its context, alternatives, and consequences. They help new team members understand WHY the system is built the way it is — not just what it does. + +## Decisions + +| # | Decision | Category | Status | Assessment | +|---|----------|----------|--------|------------| +| [0001](0001-single-shared-library-for-all-services.md) | Single shared library consumed by all services | PATTERN | Accepted | Still appropriate | +| [0002](0002-branch-name-as-sole-environment-selector.md) | Branch name as sole environment selector | PATTERN | Accepted | Still appropriate | +| [0003](0003-policy-exceptions-in-separate-whitelist-repo.md) | Policy exceptions in separate Meesho/whitelists repo | PATTERN | Accepted | Still appropriate | +| [0004](0004-fresh-whitelist-clone-per-build.md) | Whitelist cloned fresh on every build — no caching | PATTERN | Accepted | Still appropriate | +| [0005](0005-config-only-change-detection-skip-build.md) | Config-only change detection — skip build, reuse latest image | PATTERN | Accepted | Still appropriate | +| [0006](0006-ringmaster-mandatory-build-trigger-gate.md) | Ringmaster as mandatory build trigger gate | PATTERN | Accepted | Still appropriate | +| [0007](0007-gitops-via-argocd-4-step-sync-sequence.md) | GitOps via strict 4-step ArgoCD sync sequence | INFRA | Accepted | Still appropriate | +| [0008](0008-canary-mandatory-for-tier1-services-in-prd.md) | Canary deploy mandatory for Tier-1 (sp0/up0) services in prd | RELIABILITY | Accepted | Still appropriate | +| [0009](0009-jvm-heap-auto-derived-from-pod-memory-request.md) | JVM heap auto-derived from pod memory_request | RELIABILITY | Accepted | Still appropriate | +| [0010](0010-cloud-and-branch-namespaced-artifact-paths.md) | Cloud-and-branch namespaced artifact paths | DATA | Accepted | Adequate with caveats | +| [0011](0011-build-user-identity-routes-post-build-callbacks.md) | Build-user identity routes post-build callbacks | COMMUNICATION | Accepted | Adequate with caveats | +| [0012](0012-string-interpolated-helm-values-from-user-config.md) | String-interpolated Helm values from user config.yaml | COMMUNICATION | Accepted | Adequate with caveats | +| [0013](0013-multi-zone-deployables-gated-out-of-direct-jenkins-argocd.md) | Multi-zone deployables gated out of direct Jenkins ArgoCD | RELIABILITY | Accepted | Still appropriate | +| [0014](0014-open-dependabot-critical-alerts-block-builds.md) | Open Dependabot CRITICAL alerts block builds | RELIABILITY | Accepted | Still appropriate | +| [0015](0015-node-install-logic-paired-across-buildnode-groovy-and-dockerfile.md) | Node install logic paired across buildNode.groovy and Dockerfile | PATTERN | Accepted | Still appropriate | +| [0016](0016-per-environment-helm-chart-versioning.md) | Per-environment Helm chart versioning (values_v2 vs values_v3) | INFRA | Accepted | Adequate with caveats | + +## How to use + +- **New to the project?** Read these to understand why things are the way they are before touching the code. +- **Making a change?** Check if an existing ADR covers the area you're modifying — especially ADR-0003 (whitelists), ADR-0004 (whitelist caching), ADR-0007 (ArgoCD sequence order), ADR-0013 (multi-zone gating), ADR-0014 (Dependabot CRITICAL gate). +- **Making a new architectural decision?** Create a new ADR using the template in an existing file as a guide. + +## Adding a new ADR + +1. Copy an existing ADR as a template +2. Use the next sequential number (currently: 0017) +3. Fill in all sections — if you don't have info for a section, say so rather than leaving it blank +4. Get a review from the DevOps Platform team lead +5. Add a row to the table above diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..48fdd95 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,254 @@ +<!-- +Auto-generated by /meesho-init Phase 8. +Long-form architecture for devops-lib. CLAUDE.md links here but does not +duplicate the contents. +--> + +# devops-lib — Architecture + +A Jenkins shared library that every Meesho service repo consumes via +`@Library('devops-lib') _` in its `Jenkinsfile`. It encapsulates the +per-language build, the Helm/ArgoCD deploy ceremony, secret + key handling, +notification routing, and security-scan integration. + +## 1. High-level shape + +``` +service Jenkinsfile (lives in service repo) + │ + └─ buildPipeline { repo_name: ..., build_tool: ..., maintainer: ... } + │ + ┌───────────────────────────────────┘ + ▼ + vars/<entry>.groovy ← global step + │ delegates to + ▼ + src/com/meesho/stages/<stage>.groovy ← stage object + │ uses + ▼ + src/com/meesho/utilities/<helper>.groovy + │ loads + ▼ + resources/com/meesho/<template>.{yaml,Dockerfile} +``` + +### Module boundaries + +| Directory | Owns | Outside callers | +|---|---|---| +| `vars/` | Public Jenkins global steps (the "API") | Service Jenkinsfiles | +| `src/com/meesho/stages/` | Stage logic — build, deploy, notify, security scan | `vars/` | +| `src/com/meesho/utilities/` | Pure helpers — git, params, node-pool selection, templating | Stages | +| `resources/com/meesho/` | Per-language Dockerfile + Helm template + values.yaml | Stages (rendered into service workspace) | +| `resources/org/meesho/` | Jenkins agent pod templates (`*-pod.yaml`) | `vars/` (via `libraryResource`) | +| `resources/com/meesho/validate_configs.py` | Helm-values schema validator (monolithic Python) | Stages (invoked via `sh`) | + +### Architectural philosophy signals + +- **Convention over configuration.** Almost every per-service behaviour is + encoded in `vars/<entry>.groovy` + the language switch in + `buildObjHelper.groovy`. New tech stacks require new files, not new flags. +- **CPS-aware Groovy.** Helpers that touch non-serialisable Java APIs (regex + engines, template engines) sit behind `@NonCPS` boundaries + (`constructTemplate._construct`). Cross those boundaries inside a + `parallel` block and you'll get a serialisation error in production. +- **Per-call freshness vs caching.** `getWhitelistedRepos()` re-clones + `Meesho/whitelists` on every invocation (no caching). The five whitelist + checks each clone independently. This is by design: a DevOps whitelist + change must take effect on the *next* build without a library release. +- **Hard-coded `'Meesho'` org.** Many helpers embed the org as a literal — + parametrising it has knock-on effects across every service Jenkinsfile. +- **Single Python file.** `validate_configs.py` (1207 lines) holds every + CAC/values-schema rule; splitting it has been flagged + ([BUGS_AND_IMPROVEMENTS_REPORT §10](../BUGS_AND_IMPROVEMENTS_REPORT.md)). + +## 2. Entry points (vars/) + +| `vars/<file>.groovy` | Used by | Notes | +|---|---|---| +| `buildPipeline.groovy` | The legacy entry — service `Jenkinsfile` calls `buildPipeline { ... }` | Pins to `node('slave02')` (hard-coded — flagged) | +| `eksCICD.groovy` | EKS-targeted services | Calls `commonCICDFlow()` | +| `gkeCICD.groovy` | GKE-targeted services | Mirror of `eksCICD` for GCP | +| `cloudFunctionCICD.groovy` | GCP cloud-function deploys | Currently a stub (`sh 'ls -al'; echo 'Hello World'` — flagged in BUGS report) | +| `onlyPushtoJfrog.groovy` | One-off artifact push without full pipeline | JDK 11 / JDK 21 selections fall through to JDK 8 (flagged) | +| `createEKSconfigs.groovy` | Bootstrap EKS pod-template configs | | +| `gcpMigration.groovy` | One-off migration helper | | +| `log.groovy` | `log.info` wrapper used across stages | Wraps `echo` — there is no real logger | +| `stageName.groovy` | Stage-name helper | | +| `buildDockerGroovyGke.groovy` | Docker build helper for GKE | | + +## 3. Stage layer (src/com/meesho/stages/) + +`buildObjHelper.groovy:run(String build_tool)` is the dispatch switch: + +``` +build_tool → Stage class +───────────────────────────────── +maven → buildMaven +maven-* → buildMaven +gradle → buildGradle +docker → buildDocker +python-* → buildPython +node-* → buildNode +go* → buildGo +php → buildPhp +<other> → defaultBuild (silent no-op — flagged) +``` + +Other stage files (orthogonal to the build dispatch): + +- `checkOut.groovy` — Jenkins SCM checkout. Contains a method named + `chekoutSubmodule` (misspelled — flagged) at line 20. +- `deployArgoCD.groovy` — the deploy ceremony. +- `deployRingmaster.groovy` — Ringmaster / Turbo-Turtle callback. +- `deployJar.groovy` — non-container artifact deploy. +- `notify.groovy` — Slack notification + tracking-API callback. Reads + `config.maintainer` and `config.notify_channel` (defaults to + `ci-cd-status`). +- `securityScan.groovy` — invokes the in-house scanner. Hard-codes + `final String url = '172.31.5.29:63232/scans'` (P0 violation — flagged). +- `automationTest.groovy` — integration-test trigger. +- `hotFix.groovy` — sets `env.hot_fix = true` to skip Sonar / quality gate. +- `helmGenerator.groovy` — renders Helm values; line 103 contains a typo + (`catch (Exceptione)` — flagged). +- `multiBranchPipeline.groovy` — wires `GitHubSCMSource`. + +### deployArgoCD step order (load-bearing) + +``` +update_argo_repo (push new app definition) + ↓ +refresh_app_of_apps (sync app-of-apps so the new Application object exists) + ↓ +update_helm_repo (compute xms/xmx from memory_limit, push values) + ↓ +refresh_and_sync (per-service sync) +``` + +Steps 2 and 4 are not interchangeable — see [tribal-knowledge §10](tribal-knowledge.md). + +Canary enforcement (lines 408-430): if `priority_v2 ∈ {sp0, up0}` and +`envrn == 'prd'`, the deploy fails fast unless canary is properly +configured (`canary.enabled=true`, `skipAnalysis=false`, +`enableManualPromotion=true`). No whitelist or bypass. + +JVM memory (lines 220-255): `xms = xmx = memory_limit * 0.5`. There is no +`memory_request * 0.75` formula, no 64m rounding (the older claim in +tribal-knowledge has been reconciled). + +## 4. Utilities (src/com/meesho/utilities/) + +| File | Owns | +|---|---| +| `constructParam.groovy` | The big one. Loads CAC config, sets `env.BU`, `env.GCPProject`, `env.GCPLBProject`, computes `cicd_environment`, runs the five whitelist gates (`skip-sonar`, `app-config-disabled`, `multizone-enabled`, `allowedNonDevelopPrDeployment`, `ValidateCacConfig`). | +| `gitActions.groovy` | Clone, fetch, status — hard-codes the `Meesho` GitHub org in clone URLs and PR-merge URLs. | +| `nodePoolSelection.groovy` | Maps `cicd_environment` → node-pool selector. `dev`/`ftr`/`stg` all collapse to `${BU}-shared`. | +| `constructTemplate.groovy` | `@NonCPS` template engine wrapper over `SimpleTemplateEngine`. | +| `addSSHKey.groovy` | Writes a Jenkins SSH credential to `./id_github_jenkins`. **Does not** cat the key to stdout (reconciled — the historical PR #634 leak has been remediated). | +| `getDockerParams.groovy` | Helper to assemble `docker run` bindings. | +| `getYamlParameter.groovy` | Reads a single key from a YAML file. | +| `validateBuTeam.groovy` | Cross-checks the BU against team ownership. | + +## 5. Resources + +### `resources/com/meesho/` + +Templates rendered into the service workspace: + +- Per-language `Dockerfile` (`java-Dockerfile`, `go-Dockerfile`, + `node-Dockerfile`, `php-Dockerfile`, `python-{2.7,3.7,3.10.12,3.13}-Dockerfile`) +- Per-language Helm values (`go-values.yaml`, `node-values.yaml`, + `python-values.yaml`, `php-values.yaml`, `values.yaml`, `cron-values.yaml`) +- Per-language deployment manifests (`deployment.yaml`, `go-deployment.yaml`, + `node-deployment.yaml`, `php-deployment.yaml`, `python-deployment.yaml`, + `gradle-deployment.yaml`) +- ArgoCD Application template (`argoApp.yaml`) +- `Dockerfile` and `Jenkinsfile` fallbacks (rare path) +- `config.yaml` — default service shape consumed by `constructParam` +- `validate_configs.py` / `validate_configs_v2.py` — the schema validator + (v2 is the eventual replacement; both are referenced today) + +### `resources/org/meesho/` + +- `dev-pod.yaml`, `stg-pod.yaml`, `prd-pod.yaml` — Jenkins agent pod + templates loaded via `libraryResource("org/meesho/${env.INFRA_ENV}-pod.yaml")`. +- `templates/maven-3.3-jdk-8.sh`, `templates/node-12.22.sh` — bootstrap + shell scripts copied into the agent. + +## 6. Downstream services + +| Caller | Downstream | Endpoint / mechanism | Resilience | +|---|---|---|---| +| `deployArgoCD.groovy` | ArgoCD | `argocd login ${env.argoURL}:443 --grpc-web`, `argocd app sync`, `argocd app refresh` | `--http-retry-max 3 --retry-backoff-duration 1m` | +| `deployRingmaster.groovy` | Ringmaster *or* Turbo-Turtle | `POST http://turbo-turtle.meeshogcp.in/...` (chosen by `getUserId() == "ringmaster-bot"`) | None — direct `curl` | +| `notify.groovy` | Slack | `slackSend channel: ..., message: ...` | None | +| `notify.groovy` | Deployment Tracker | `POST https://deployment-tracker.meeshoint.in/...` (and `.prd.meesho.int`) | None | +| `buildMaven.groovy` / `buildNode.groovy` / `buildGo.groovy` | JFrog | `mvn deploy` / `npm publish` / artifact upload | Branch-gated: master, main, gcp-main, gcp-master | +| `buildMaven.groovy` / `buildGradle.groovy` etc. | S3 | `aws s3 cp …` | Same branch gate | +| `buildNode.groovy` | Docker registry | `docker push` | `retryDockerPush` retry wrapper | +| `buildMaven.groovy` etc. | SonarQube | `withSonarQubeEnv { … }` → `sonarqube-prd` | Skipped via `skip-sonar-whitelist.yaml` for Maven prd | +| `constructParam.groovy` | Vault | `vault-prd.meeshogcp.in`, `vault-dev.meeshogcp.in` | None | +| `securityScan.groovy` | In-house scanner | `POST http://172.31.5.29:63232/scans` (P0 — should be DNS, flagged) | None | + +## 7. Critical invariants & gotchas + +(Most are also enumerated in [docs/tribal-knowledge.md](tribal-knowledge.md); +this section captures the ones that change the *shape* of the code.) + +1. **`config.yaml` is read once by `constructParam.run()`.** Everything + downstream reads from `env.*` it set. Don't introduce a second config + read; mutate `env.*` instead. +2. **`hot_fix` shortcuts.** `env.hot_fix = true` (set by `hotFix.groovy:11`) + skips Sonar, quality gates, and several validation steps in + `buildMaven` / `buildGo`. Use it deliberately, not as a "skip everything" + knob. +3. **`branch_name = 'repo'`** appears as a hard-coded string in + `buildGradle.groovy:252,285` (flagged in BUGS report). Do not assume + `branch_name` is dynamic — the comparison against `'master'/'main'` + never matches there. +4. **`rm -rf *`** appears in six locations across `buildPython`, `buildMaven`, + `buildGradle` (flagged). Be aware of the working directory when adding + stages near these — there is no directory guard. +5. **`generic catch (Exception e)`** is used ~87 times across stages + (flagged). New code should prefer typed exceptions, but existing handlers + suppress everything — be cautious assuming a stage "succeeded". + +## 8. Configuration touch points + +| Config | Where read | Drives | +|---|---|---| +| `config.bu` | `constructParam:178,255,308,318` | Helm chart path, ArgoCD namespace, GCP project name | +| `env.INFRA_ENV` | `eksCICD:57`, `createEKSconfigs:5`, `onlyPushtoJfrog:4` | Jenkins agent pod template | +| `env.CHANGE_ID` | `constructParam:107-110` | PR vs branch detection; `cicd_environment` remap (`prd→int` for main/master PRs, `stg→ftr` for develop PRs) | +| `env.hot_fix` | `buildGo:25-28`, `buildMaven:37-40`, `hotFix:11` | Skip Sonar / quality gate | +| `param.build_tool` | `buildObjHelper:run`, `vars/buildPipeline:12` | Stage class selection | +| `param.maintainer` | `notify:9,31` | Slack mention | +| `param.skip_test` | `buildMaven:17,206-208` | Maven `-DskipTests` | +| `param.skip_sonar` | `buildMaven:18,240-243` | Sonar bypass | +| `param.skip_security_scan` | `securityScan:7-9` | Security scan bypass | +| `param.skip_notify` | `notify:10,26-28` | Slack bypass | +| `param.notify_channel` | `notify:11` | Slack channel (default `ci-cd-status`) | +| `param.push_to_jfrog` | `buildMaven:19,377` | Allow non-default-branch JFrog push | +| `param.push_to_s3` | `buildMaven:20,462` | Allow non-default-branch S3 push | + +## 9. What's missing (and known) + +`BUGS_AND_IMPROVEMENTS_REPORT.md` is the authoritative catalogue. Highlights +that affect *how* you should approach changes: + +- **No test suite** (P0). Don't fabricate test commands. +- **No retry on most stages** (P1) — only ArgoCD sync and Docker push retry. +- **Hard-coded IPs** (P0) — `securityScan.groovy:11` is a known violation + pending remediation. +- **Inconsistent logging** — mix of `log.info()` (which is a thin wrapper) + and bare `echo`. Prefer `log.info` for new code. +- **Commented-out blocks** — `buildPython:13-38`, `buildNode:27-31`, + `buildGradle:470-477` carry large dead sections. Don't extend them; if a + block is genuinely dead, delete it in a separate PR. + +## See also + +- [`docs/acronyms.md`](acronyms.md) +- [`docs/tribal-knowledge.md`](tribal-knowledge.md) +- [`../review-learnings.md`](../review-learnings.md) +- [`../BUGS_AND_IMPROVEMENTS_REPORT.md`](../BUGS_AND_IMPROVEMENTS_REPORT.md) diff --git a/docs/downstreams.md b/docs/downstreams.md new file mode 100644 index 0000000..4ca8cb2 --- /dev/null +++ b/docs/downstreams.md @@ -0,0 +1,115 @@ +# Downstream services + +> External APIs and services called by the devops-lib pipeline stages. Infrastructure (storage, registries) lives in [infrastructure.md](infrastructure.md). + +## ArgoCD + +Used by `deployArgoCD.groovy` to sync Helm releases to GKE clusters. + +| Environment | URL | Credentials | +|-------------|-----|-------------| +| `prd` | `argocd-{bu}-prd.meeshogcp.in` | `argocd-{bu}-prd-creds` | +| `int` | `argocd-shared-int.meeshogcp.in` | `argocd-shared-int-creds` | +| `stg` / `ftr` | `argocd-dev.meeshogcp.in` | `argocd-dev-creds` | + +Calls: `argocd app sync <app-name>` via CLI authenticated against the above URLs. + +## Ringmaster + +Used by `deployRingmaster.groovy` and `notify.groovy` for deployment tracking and the CD approval portal. + +| Environment | API base URL | +|-------------|-------------| +| `prd` / `int` | `https://ringmaster-api.meeshogcp.in` | +| `stg` / `ftr` | `https://ringmaster-api.admin.meeshogcp.in` | + +Endpoints called: +- `POST /api/v1/key/cicd/cd/update?workingEnv={env}` — build result + image tag + applications deployed +- `POST /api/v1/key/update/deployment-history?workingEnv={env}` — deployment history (prd branch builds only) + +Credentials: Jenkins credential `ringmaster-token` (username + password). + +## Turbo-Turtle (new CICD callback) + +Used by `deployRingmaster.groovy` for builds triggered by users other than `ringmaster-bot`. + +| Environment | Base URL | +|-------------|---------| +| `prd` / `int` | `http://turbo-turtle.meeshogcp.in` | +| `stg` / `ftr` | `http://turbo-turtle.admin.meeshogcp.in` | + +Endpoint called: `POST /api/v1/ci/jenkins/callback` + +## Deployment Tracker (legacy) + +Used by `notify.groovy` for prd branch builds. + +| Cloud | URL | +|-------|-----| +| GCP | `http://deployment-tracker.prd.meesho.int` | +| AWS | `http://deployment-tracker.meeshoint.in` | + +Endpoint called: `POST /api/1.0/deployment-tracker/jenkins/create` + +## SonarQube + +Used by all build stages for code quality gating. + +| Environment | URL | Token credential | +|-------------|-----|-----------------| +| `prd` / `int` | `https://sonarqube-prd.meeshogcp.in` | `sonar-token-prod` | +| `stg` / `ftr` | `https://sonarqube-{bu}-dev.meeshogcp.in` | `sonar-token-{bu}-dev` | + +## Vault + +Used by build stages to retrieve service secrets at deploy time. + +| Environment | URL | Token credential | +|-------------|-----|-----------------| +| `prd` / `int` | `https://vault-prd.meeshogcp.in` | `vault-prd-token` | +| `stg` / `ftr` | `https://vault-dev.meeshogcp.in` | `vault-dev-token` | + +## JFrog Artifactory + +Used by `buildMaven.groovy` and `onlyPushtoJfrog.groovy` for Maven JAR publishing and resolution. + +Credential: `svc-devops-meesho` (env `GITHUB_CRED`). + +## GitHub — Meesho/whitelists + +Cloned at runtime by `constructParam.groovy` to fetch per-repo policy YAML files. + +``` +url: https://github.com/Meesho/whitelists.git +branch: main +credentialsId: cicd-github-app +``` + +Files read: +- `skip-sonar-whitelist.yaml` +- `app-config-disabled.yaml` +- `multizone-enabled-repos.yaml` +- `allowedNonDevelopPrDeploymentToInt.yaml` +- `ValidateCacConfig.yaml` + +## GitHub — devops-helm-charts / devops-argo-config + +Cloned by `deployArgoCD.groovy` to update Helm values and ArgoCD app manifests. + +| Repo | Branch (prd) | Branch (stg) | +|------|-------------|-------------| +| `devops-helm-charts` | `main` | `develop` | +| `devops-argo-config` | `main` | `develop` | + +Credential: `svc-devops-meesho`. + +## Athens (Go module proxy) + +Used by `buildGo.groovy` via `GOPRIVATE=github.com/Meesho` + `GOPROXY`. + +| Environment | URL | +|-------------|-----| +| `prd` / `int` | `https://athens-prd.meeshogcp.in` | +| `stg` / `ftr` | `https://athens-dev.meeshogcp.in` | + +<!-- meesho-init: generated-at=2026-05-07T08:39:42Z base-sha=805350dfbd5b354663e8e2a90bfc219c4c267e00 --> diff --git a/docs/golden-prs.md b/docs/golden-prs.md new file mode 100644 index 0000000..c79751f --- /dev/null +++ b/docs/golden-prs.md @@ -0,0 +1,108 @@ +# Golden PRs — devops-lib + +Curated benchmark PRs for evaluating LLM coding agents on this service. +Selected on 2026-05-07 from the Golden PR Selection pipeline (16 from +the suggested set + 0 user-added). + +## Picks (16) + +### #504 — Include common priority(CP) in dependabot blocking logic +- URL: https://github.com/Meesho/devops-lib/pull/504 +- Author: @dhyey-meesho +- Stratum: feature +- Merged: 2025-05-09 + +### #616 — Feat/pbac enabled flag support +- URL: https://github.com/Meesho/devops-lib/pull/616 +- Author: @amansrivastava118 +- Stratum: feature +- Merged: 2025-11-04 + +### #643 — go sonar scans +- URL: https://github.com/Meesho/devops-lib/pull/643 +- Author: @AryamanParida-Meesho +- Stratum: feature +- Merged: 2026-01-13 + +### #664 — Add repoType handling to build scripts +- URL: https://github.com/Meesho/devops-lib/pull/664 +- Author: @vg-meesho +- Stratum: feature +- Merged: 2026-02-05 + +### #687 — Ft/toolchain integration mvn pyth +- URL: https://github.com/Meesho/devops-lib/pull/687 +- Author: @AryamanParida-Meesho +- Stratum: feature +- Merged: 2026-02-25 + +### #694 — sidecar cotnaienr +- URL: https://github.com/Meesho/devops-lib/pull/694 +- Author: @AryamanParida-Meesho +- Stratum: feature +- Merged: 2026-03-05 + +### #710 — Node changes +- URL: https://github.com/Meesho/devops-lib/pull/710 +- Author: @AryamanParida-Meesho +- Stratum: feature +- Merged: 2026-04-08 + +### #730 — Update validate_configs.py +- URL: https://github.com/Meesho/devops-lib/pull/730 +- Author: @abhinandanv13-meesho +- Stratum: feature +- Merged: 2026-04-14 + +### #667 — secret path fix for node applcaiton +- URL: https://github.com/Meesho/devops-lib/pull/667 +- Author: @ShrutiKoshta-meesho +- Stratum: bugfix +- Merged: 2026-02-05 + +### #696 — Mq before vs +- URL: https://github.com/Meesho/devops-lib/pull/696 +- Author: @AryamanParida-Meesho +- Stratum: bugfix +- Merged: 2026-03-04 + +### #711 — Configure Git to use SSH for GitHub URLs in Dockerfile +- URL: https://github.com/Meesho/devops-lib/pull/711 +- Author: @sahil-meesho +- Stratum: bugfix +- Merged: 2026-03-24 + +### #713 — node-fix +- URL: https://github.com/Meesho/devops-lib/pull/713 +- Author: @ShrutiKoshta-meesho +- Stratum: bugfix +- Merged: 2026-03-26 + +### #716 — Update Dockerfile to configure SSH for GitHub access +- URL: https://github.com/Meesho/devops-lib/pull/716 +- Author: @sahil-meesho +- Stratum: bugfix +- Merged: 2026-03-30 + +### #702 — adding command level +- URL: https://github.com/Meesho/devops-lib/pull/702 +- Author: @AryamanParida-Meesho +- Stratum: refactor +- Merged: 2026-03-11 + +### #721 — Remove 2 mvn clean +- URL: https://github.com/Meesho/devops-lib/pull/721 +- Author: @mahak4jain +- Stratum: refactor +- Merged: 2026-04-07 + +### #727 — Refactor npm install command in Dockerfile and buildNode.groovy +- URL: https://github.com/Meesho/devops-lib/pull/727 +- Author: @yeleswaramteja +- Stratum: refactor +- Merged: 2026-04-17 + + +--- +*Generated by `meesho-golden-pr-register` on 2026-05-07.* +*Source run hash: `c8e9537048ec599b`.* diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..33edfbc --- /dev/null +++ b/docs/index.md @@ -0,0 +1,43 @@ +# Documentation Index + +> Catalogue of every doc reachable from this repo. CLAUDE.md's `## Required reading` lists the must-reads; everything else lives here. Auto-generated by `/meesho-init` Phase 10 — edits to the body are reset on next run. To add a doc, drop the file into `docs/` and re-run `/meesho-init`. + +## Authoritative + +| Doc | What it covers | +| --- | --- | +| [docs/architecture.md](architecture.md) | System design, module boundaries, deployArgoCD 4-step ceremony, downstream services, critical invariants | +| [docs/wiki/index.md](wiki/index.md) | Synthesized concept pages — owned by `/m-wiki` | +| [docs/adr/README.md](adr/README.md) | Architecture Decision Records (16 ADRs, retroactively captured) — owned by `/m-docs:adr-generator` | + +## Reference + +| Doc | What it covers | +| --- | --- | +| [docs/acronyms.md](acronyms.md) | Domain-specific acronyms used in this repo's code and docs (BU, BUILDKIT, CAC, GCPP, INFRA, …) | +| [docs/downstreams.md](downstreams.md) | External APIs and services called by the devops-lib pipeline stages (ArgoCD, Ringmaster, JFrog, SonarQube, Vault, Slack, Deployment Tracker) | +| [docs/golden-prs.md](golden-prs.md) | Curated benchmark PRs for evaluating LLM coding agents on this service | +| [docs/infrastructure.md](infrastructure.md) | Storage and registry infrastructure used by the devops-lib pipeline per environment (GAR, ECR, S3, GCS) | +| [docs/review-learnings.md](review-learnings.md) | PR-review-derived skill proposals — auto-generated by `/m-docs:pr-learnings` (different from `review-learnings.md` at repo root, which is the manual learnings log) | +| [docs/SECURITY.md](SECURITY.md) | Security policy and contact — auto-generated by `/m-docs:security-init` | +| [docs/tribal-knowledge.md](tribal-knowledge.md) | Non-obvious conventions, design decisions, operational patterns (15 numbered sections — load-bearing reads for anyone touching the pipeline) | + +## Skills proposals + +| Doc | What it covers | +| --- | --- | +| [docs/skills/README.md](skills/README.md) | Index of proposed Claude Code skills for devops-lib | +| [docs/skills/pipeline-tracer/](skills/pipeline-tracer/) | Trace a service's complete execution path through devops-lib | +| [docs/skills/library-impact-analyzer/](skills/library-impact-analyzer/) | Identify build-tool / stage / consumer blast radius for a devops-lib PR | +| [docs/skills/build-failure-debugger/](skills/build-failure-debugger/) | Map a Jenkins job log to the exact devops-lib code path that produced the failure | + +## Other (outside `docs/`) + +| Doc | What it covers | +| --- | --- | +| [README.md](../README.md) | Service blurb + `config.yaml` schema + adding-a-build-stage walkthrough | +| [CLAUDE.md](../CLAUDE.md) | Claude-facing repo guide — must-read for agents working here | +| [review-learnings.md](../review-learnings.md) | Manual PR-review learnings log (root-level — note: distinct from `docs/review-learnings.md`, which is the auto-generated skill proposals) | +| [BUGS_AND_IMPROVEMENTS_REPORT.md](../BUGS_AND_IMPROVEMENTS_REPORT.md) | Known bug + tech-debt catalogue — read before "fixing" anything that looks suspect | + +<!-- meesho-init: plugin-version=1.0.33-with-phase10-backport generated-at=2026-05-22T20:00:00Z base-sha=e504a2b428e4 --> diff --git a/docs/infrastructure.md b/docs/infrastructure.md new file mode 100644 index 0000000..7cbe8d7 --- /dev/null +++ b/docs/infrastructure.md @@ -0,0 +1,67 @@ +# Infrastructure + +> Storage and registry infrastructure used by the devops-lib pipeline per environment. External service dependencies live in [downstreams.md](downstreams.md). + +Note: `devops-lib` is a Jenkins Shared Library — it does not own databases, caches, or message queues. The infrastructure listed here is used transiently during pipeline execution (artifact storage, container registry, Docker daemon). + +## Artifact storage + +Built JARs and other build outputs are stored in object storage before being referenced at deploy time. + +| Type | Environment | Bucket / Path | +|------|-------------|--------------| +| GCS | `prd` / `int` | `gcs-infr-dvps-meesho-artifacts-prd` / `gcs-infr-dvps-meesho-artifacts-int` | +| GCS | `stg` | `gcs-infr-dvps-meesho-artifacts-stg` | +| GCS | `ftr` | `gcs-infr-dvps-meesho-artifacts-ftr` | +| S3 (AWS prd) | `prd` / `int` | `meesho-prod-artifacts` (ap-southeast-1, account `847438129436`) | +| S3 (AWS dev) | `stg` / `ftr` | `meesho-stg-artifacts` (ap-south-1, account `766380763301`) | + +Set as `env.objBucket` by `constructParam.run()`. + +## Container registry + +Docker images are built and pushed here by all build stages. + +| Type | Registry URL | Used for | +|------|-------------|---------| +| GAR (GCP) | `asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622` | All GCP builds — push target | +| GAR admin repo | `asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin` | Build-time base images | +| ECR (AWS prd) | `847438129436.dkr.ecr.ap-southeast-1.amazonaws.com` | AWS prd / int | +| ECR (AWS dev) | `766380763301.dkr.ecr.ap-south-1.amazonaws.com` | AWS stg / ftr | + +Set as `env.registry` / `env.buildRegistry` by `constructParam.run()`. + +## Docker daemon (dind) + +Build stages connect to a remote Docker daemon rather than a local socket. The host is injected as `env.DOCKER_HOST`. + +| Environment | Docker host | +|-------------|------------| +| `prd` | `dind-prd-svc` | +| `int` | `dind-int-svc` | +| `stg` / `ftr` | `dind-dev-new-svc.jenkins-new.svc.cluster.local` | + +## Jenkins agent pods (GCP) + +GCP builds run in Kubernetes pods defined in `resources/org/meesho/`. Pod specs are referenced by environment: + +| Environment | Pod spec file | +|-------------|--------------| +| `prd` | `resources/org/meesho/prd-pod.yaml` | +| `int` | `resources/org/meesho/int-pod.yaml` | +| `stg` | `resources/org/meesho/stg-pod.yaml` | +| `ftr` | `resources/org/meesho/ftr-pod.yaml` | + +The `container('devops-tools')` container inside these pods runs all pipeline stages. + +## GCP projects + +| Environment | GCP project | +|-------------|------------| +| `prd` | `meesho-{bu}-prd-0622` | +| `int` | `meesho-shared-int-0525` | +| `stg` / `ftr` | `meesho-{bu}-dev-0622` | + +Set as `env.GCPProject` by `constructParam.run()`. + +<!-- meesho-init: generated-at=2026-05-07T08:39:42Z base-sha=805350dfbd5b354663e8e2a90bfc219c4c267e00 --> diff --git a/docs/review-learnings.md b/docs/review-learnings.md new file mode 100644 index 0000000..2cece69 --- /dev/null +++ b/docs/review-learnings.md @@ -0,0 +1,27 @@ +<!-- auto-generated by pr-learnings --> + +## buildNode.groovy: pnpm must be globally installed before invoking pnpm commands + +**Takeaway:** When auto-selecting the install command in `buildNode.groovy` based on lockfile presence, pnpm must be installed globally first (`npm install -g pnpm@<version>`) — emitting `pnpm install --frozen-lockfile` alone will fail in environments where pnpm is not on the PATH. + +**Pattern:** Any time `buildNode.groovy` detects a `pnpm-lock.yaml` and constructs an `npm_install_arg` that invokes `pnpm`, the command string must prefix with a global install step. The version should be pinned to avoid silent drift. + +**Rationale:** Jenkins agents running Node builds do not have pnpm pre-installed. The auto-selection logic that branches on `fileExists("${repo_name}/pnpm-lock.yaml")` is responsible for producing a self-contained, executable command string — if pnpm is absent from the agent PATH the build fails with a command-not-found error. Reviewers also flagged that lock-file cleanup helpers should cover `pnpm-lock.yaml` alongside `package-lock.json`. + +**Concrete example** (from [PR #727](https://github.com/Meesho/devops-lib/pull/727), `src/com/meesho/stages/buildNode.groovy:L193`): + +```groovy +// Before (incomplete — pnpm not guaranteed on PATH) +if (fileExists("${repo_name}/pnpm-lock.yaml")) { + npm_install_arg = 'pnpm install --frozen-lockfile' +} + +// After (reviewer-suggested correction) +if (fileExists("${repo_name}/pnpm-lock.yaml")) { + npm_install_arg = 'npm install -g pnpm@10.33.0 && pnpm install --frozen-lockfile' +} +``` + +**Examples from PRs:** +- [PR #727](https://github.com/Meesho/devops-lib/pull/727): Auto-selection logic added for pnpm/npm-ci/npm-install; reviewer caught that pnpm must be globally installed first. +- [PR #728](https://github.com/Meesho/devops-lib/pull/728): Reviewer noted cleanup helpers should also handle `pnpm-lock.yaml`. diff --git a/docs/skills/README.md b/docs/skills/README.md new file mode 100644 index 0000000..b648add --- /dev/null +++ b/docs/skills/README.md @@ -0,0 +1,9 @@ +# devops-lib Skill Proposals + +Proposed Claude Code skills for devops-lib. Each proposal targets a specific daily friction point observed from code review patterns, task-replay data, and tribal knowledge gaps. + +| Skill | One-line description | Priority | Owner | +|---|---|---|---| +| [`pipeline-tracer`](./pipeline-tracer/SPEC.md) | Given a service config.yaml and branch/trigger context, trace the complete execution path through devops-lib — every stage, every decision point, every policy check — with exact code locations | High | DevOps Platform | +| [`library-impact-analyzer`](./library-impact-analyzer/SPEC.md) | Given a devops-lib PR, identify which build_tool types, pipeline stages, and consumer service categories are affected — so reviewers know the blast radius before merging | High | DevOps Platform | +| [`build-failure-debugger`](./build-failure-debugger/SPEC.md) | Given a Jenkins job log from a failed devops-lib build, identify exactly which stage failed, trace the code path through devops-lib that produced the failure, and output a concrete fix | High | DevOps Platform | diff --git a/docs/skills/build-failure-debugger/SPEC.md b/docs/skills/build-failure-debugger/SPEC.md new file mode 100644 index 0000000..c5acb82 --- /dev/null +++ b/docs/skills/build-failure-debugger/SPEC.md @@ -0,0 +1,129 @@ +# Skill: build-failure-debugger + +**One-line description:** Given a Jenkins job log from a failed devops-lib build, identify exactly which stage failed, trace the code path through devops-lib that produced the failure, and output a concrete fix with the exact file and line driving the error. + +**Owner:** DevOps Platform team + +--- + +## Why this skill exists + +Jenkins job logs are long, noisy, and mixed-language (Groovy stack traces, Maven output, Docker build output, ArgoCD sync output). When a build fails, an engineer must: + +1. Scroll through 2000+ lines to find the failure point +2. Know that "Build" maps to `buildMaven.groovy`, "Deploying to ArgoCD" maps to `deployArgoCD.groovy`, etc. +3. Know which code path was taken based on `build_tool` and `cicd_environment` +4. Know which external system (GAR, SonarQube, ArgoCD, Helm repo) caused the failure vs which devops-lib logic caused it + +The mapping from Jenkins stage names to devops-lib source files is non-obvious and not documented anywhere. A "SonarQube Quality Gate" failure could mean: the threshold was breached, the project doesn't exist in Sonar yet, or a whitelist entry is missing. Three different fixes, identical log output. + +This skill knows the stage-to-file mapping, the code paths for each failure mode, and the set of conditions that trigger each error. + +--- + +## Trigger + +- User pastes a Jenkins job log (or the relevant failure excerpt) +- `"this Jenkins build failed, help me debug it"` with log attached +- `"sonar quality gate failing for auth-service on develop"` +- `"build stuck at ArgoCD sync for catalog-service in prd"` +- Proactively: when a user shares a build URL, fetch the log and diagnose + +--- + +## Input + +- Jenkins job log (pasted text or URL to the Jenkins job) +- Optionally: the service `config.yaml` (to know `build_tool`, `skip_sonar`, `deployArgo`, etc.) + +The skill can infer build_tool and environment from the log itself if config.yaml is not provided (Jenkins prints `build_tool` and `cicd_environment` at pipeline start via `constructParam.run()`). + +--- + +## Expected output + +Given: Jenkins log showing failure in "Deploying to ArgoCD" stage for `catalog-service` on `develop` + +``` +Build Failure Debug — catalog-service | branch: develop | env: stg + +Stage failed: Deploying to ArgoCD +devops-lib: src/com/meesho/utilities/deployArgoCD.groovy + +Step that failed: update_helm_repo (step 3 of 4) +Error in log: ERROR: Helm push failed — chart version 0.1.142 already exists + +Root cause: The Helm chart version was not bumped between this push and the previous + build. deployArgoCD.groovy uses the image SHA as the chart version suffix, but the + SHA collision occurred because the source didn't change (config-only build — latest + tag was reused from GAR without a new image build). + +Fix: Force a source change to generate a new image SHA, OR bump the chart version + manually in devops-argo-config. + +deployment_order check: 'catalog-service-stg' — verify this matches the app name + in devops-argo-config (wrong name → silent sync failure at step 4, not step 3). + +Relevant code: + deployArgoCD.groovy:update_helm_repo() — pushes chart with SHA-derived version + constructParam.groovy:getConfigOnlyChange() — sets SKIP_BUILD flag when only YAMLs changed +``` + +--- + +## Stage-to-file mapping (built-in knowledge) + +| Jenkins stage name | devops-lib file | +|---|---| +| `Build` | `buildMaven.groovy` / `buildGo.groovy` / `buildNode.groovy` / etc. (by `build_tool`) | +| `Docker Build & Push` | `buildMaven.groovy:dockerBuildAndPush()` / equivalent in each build stage | +| `SonarQube Analysis` | `buildMaven.groovy:sonar_scan()` / `buildGo.groovy:sonar_scan()` | +| `SonarQube Quality Gate` | `constructParam.groovy:waitForQualityGate()` | +| `CAC Validation` | `buildMaven.groovy:cac_validation()` / `buildGo.groovy:cac_validation()` | +| `Deploying to ArgoCD` | `deployArgoCD.groovy` (4-step: update_argo_repo → refresh_app_of_apps → update_helm_repo → refresh_and_sync) | +| `Notify` | `notify.groovy` / `notifySlack.groovy` | +| `AppConfig Validation` | `constructParam.groovy:validateAppConfig()` | + +--- + +## Failure mode taxonomy + +The skill classifies every failure into one of these categories before diagnosis: + +| Category | Signal in log | devops-lib location | +|---|---|---| +| Build compilation | `BUILD FAILURE` / `go build failed` | build<Lang>.groovy | +| Docker push to GAR | `denied` / `UNAUTHORIZED` on push | build<Lang>.groovy:dockerBuildAndPush() | +| Sonar threshold | `Quality Gate status: FAILED` | constructParam.groovy:waitForQualityGate() | +| Sonar project missing | `Project not found` | buildGo.groovy:sonar_scan() auto-create logic | +| ArgoCD step 1 (argo_repo update) | `git push failed` in argo-config repo | deployArgoCD.groovy:update_argo_repo() | +| ArgoCD step 2 (app-of-apps refresh) | `app not found` / `no Application object` | deployArgoCD.groovy:refresh_app_of_apps() — new service, ArgoCD app not yet created | +| ArgoCD step 3 (helm push) | `chart version already exists` | deployArgoCD.groovy:update_helm_repo() | +| ArgoCD step 4 (sync) | `OutOfSync` / `Helm values error` | deployArgoCD.groovy:refresh_and_sync() | +| Wrong deployment_order | Silent sync on wrong app name | config.yaml:deployment_order vs devops-argo-config | +| Multizone gate | `must be deployed via Ringmaster` | constructParam.groovy:isMultizoneEnabled() | + +--- + +## Dependencies + +- Full read access to devops-lib source (stage-to-file mapping, failure message strings) +- Jenkins log (pasted by engineer or fetched via Jenkins API) +- Optionally: `Meesho/whitelists` (to check if skip_sonar or multizone entries explain the failure) +- Optionally: `Meesho/devops-argo-config` (to validate `deployment_order` app names for ArgoCD failures) + +--- + +## Design notes + +- The skill must handle truncated logs (Jenkins UI often shows the last N lines). It should ask for the full log if the failure point is not visible. +- For hotfix builds: sonar and quality gate failures are expected to be suppressed; if they appear, the hotfix path was not taken — the skill should check the branch name pattern. +- The skill should distinguish between a devops-lib bug (code path is wrong) vs a configuration error (wrong value in config.yaml or whitelist) vs an external system error (GAR down, Sonar unreachable). + +--- + +## Open questions + +- Should it auto-fetch the Jenkins log via the Jenkins API if given a job URL? +- For ArgoCD step 4 failures (Helm values schema errors): should it parse the exact Helm error and cross-reference with the deployment.yaml schema? +- Should it suggest a rerun command or ArgoCD force-sync as a recovery action? diff --git a/docs/skills/library-impact-analyzer/SPEC.md b/docs/skills/library-impact-analyzer/SPEC.md new file mode 100644 index 0000000..a0a6c34 --- /dev/null +++ b/docs/skills/library-impact-analyzer/SPEC.md @@ -0,0 +1,73 @@ +# Skill: library-impact-analyzer + +**One-line description:** Given a devops-lib PR, identify which build_tool types, pipeline stages, and consumer service categories are affected — so reviewers know the blast radius before merging. + +**Owner:** DevOps Platform team + +--- + +## Why this skill exists + +devops-lib is consumed via `@Library('devops-lib@main')` by every Meesho microservice. A change to a shared file like `constructParam.groovy` or `buildObjHelper.groovy` can silently affect hundreds of services across all build types and environments. There is currently no tooling to answer the basic pre-merge question: **"what does this change actually affect?"** + +Common dangerous patterns caught too late: +- A change to `constructParam.groovy:run()` that's tested on maven but breaks node builds (different code path) +- A change to `buildGo.groovy` that fixes prd but changes stg behavior (env-conditional logic) +- A change to `eksCICD.groovy:allowedUsers` that accidentally narrows who can trigger builds + +The review process today is manual — a senior DevOps engineer reads the diff and mentally simulates which build types it touches. This is error-prone and doesn't scale as the library grows. + +--- + +## Trigger + +- PR opened against devops-lib (automatic on any PR touching `src/`, `vars/`, or `resources/`) +- Manual: `"what does this PR affect?"` with a PR number or diff pasted +- `"impact analysis for PR #643"` + +--- + +## Expected output + +Given PR touching `src/com/meesho/utilities/constructParam.groovy` and `src/com/meesho/stages/buildGo.groovy`: + +``` +Impact Analysis — PR #643 + +Files changed: constructParam.groovy, buildGo.groovy + +Affected build paths: + ✦ constructParam.groovy is called by ALL build types on EVERY build + → Changes here affect: maven, go, node-*, python-*, gradle, php, rust, docker + → Changes here affect: ALL environments (prd, stg, int, ftr) + Changed method: skipSonarCheckForGo() — new method, additive, low risk + + ✦ buildGo.groovy affects: go, go-1.22, go-1.21 (any build_tool matching /^go.*/) + → Environments: prd + stg (sonar_scan() only runs in these envs) + Changed: sonar_scan() — adds exclusion logic and auto-project creation + +Risk assessment: + constructParam.groovy: LOW (new method only, no existing method modified) + buildGo.groovy: MEDIUM (modifies sonar_scan() which runs in prd) + +Suggested test coverage before merge: + □ Trigger a Go service build in stg to verify sonar exclusions work + □ Trigger a Maven service build to confirm constructParam changes are neutral + □ Check sonar_scan() does not break for services without sonar-project.properties +``` + +--- + +## Dependencies + +- Reads devops-lib source to build a call graph (which methods call what, which build_tool routes to which stage) +- `git diff` of the PR (from `gh pr diff <number>` or GitHub API) +- Optional: `Meesho/whitelists` to identify which consumer repos are on relevant whitelists + +--- + +## Open questions + +- Should it post the impact analysis as a PR comment automatically (requires GitHub token), or output to stdout? +- Should it attempt to enumerate actual consumer services affected (requires access to consumer repos), or stop at build_tool categories? +- Should it flag changes to `allowedUsers` lists as HIGH risk automatically (any change to who can trigger builds is sensitive)? diff --git a/docs/skills/pipeline-tracer/SPEC.md b/docs/skills/pipeline-tracer/SPEC.md new file mode 100644 index 0000000..85e8b9c --- /dev/null +++ b/docs/skills/pipeline-tracer/SPEC.md @@ -0,0 +1,112 @@ +# Skill: pipeline-tracer + +**One-line description:** Given a service's config.yaml and a branch/trigger context, trace the complete execution path through devops-lib — every stage, every decision point, every policy check — and output a human-readable flow with the exact code locations driving each step. + +**Owner:** DevOps Platform team + +--- + +## Why this skill exists + +devops-lib's pipeline is a routing tree, not a linear script. A single `eksCICD` call dispatches to different stages based on `build_tool`, takes different paths based on `env.BRANCH_NAME` and PR target, checks multiple whitelists, conditionally runs sonar, CAC validation, ArgoCD sync, and sends different notifications depending on the environment. The full execution path for a given service in a given context spans 15+ files. + +No one has a complete mental model of this tree for every service type. Consequences: +- Engineers add a whitelist exception but don't know which of the 3 sonar-skip checks it actually bypasses +- Reviewers approve a stage change without knowing it only runs in `prd` (not `stg`) +- New team members spend days understanding why their build skips certain stages +- Debugging requires mentally simulating the entire dispatch chain from `eksCICD.groovy` down + +This skill is the complement to `library-impact-analyzer`: impact-analyzer answers "what does a code change affect?" — pipeline-tracer answers "for this specific service in this specific context, what exact path does the code take?" + +--- + +## Trigger + +- `"trace pipeline for payment-service on develop branch"` +- `"what stages run for a hotfix build of catalog-service?"` +- `"why is sonar being skipped for auth-service?"` +- `"show me the full pipeline path for a PR from feature/x to main in supply-chain-service"` +- Proactively: attached to `service-onboarder` output — show the new service's expected pipeline before its first build + +--- + +## Input + +- A service `config.yaml` (file path or pasted content) — provides `build_tool`, `team`, `bu`, `skip_sonar`, `deployArgo`, `deployment_order`, etc. +- A trigger context: branch name (`develop`, `main`, `hotfix/x`, `feature/y`) and optionally a PR target (`main` or `develop`) + +--- + +## Expected output + +Given: `config.yaml` for `payment-service` (build_tool: maven, bu: supply), branch: `develop` + +``` +Pipeline Trace — payment-service | branch: develop | env: stg + +Entry point: vars/eksCICD.groovy +Trigger check: PASS — develop branch, no PR target → env = stg + +Pod selection: resources/org/meesho/stg-pod.yaml + image: build-tools:lunar-v2.0.21 + node pool: supply-shared (BU-scoped stg pool) + +Build stage: src/com/meesho/stages/buildMaven.groovy + build_tool 'maven' → buildObjHelper → buildMaven + Config-only change check: RUNS (skips build if only *.yaml changed) + Docker image: stg/payments/payment-service:<sha> + +Policy checks: + skip_sonar: false → sonar WILL run + skip-sonar-whitelist: payment-service NOT on list → sonar runs + CAC validation: payment-service on ValidateCacConfig whitelist → RUNS + appConfigEnabled: true → AppConfig validation RUNS + multizone: payment-service NOT on multizone list → deploy proceeds normally + +ArgoCD deploy: + deployArgo: true → WILL deploy + deployment_order: [payment-service] + 4-step sequence: update_argo_repo → refresh_app_of_apps → update_helm_repo → refresh_and_sync + ArgoCD app: payment-service-stg + +Notification: + notify_channel: #payments-alerts + Ringmaster callback: NO (stg build, not prd) + Turbo-Turtle callback: YES (stg deploy confirmation) + +Total stages: 7 | Estimated duration: 12–18 min +``` + +--- + +## Dependencies + +- Full read access to devops-lib source (the skill builds a live call graph from the source) +- `Meesho/whitelists` read access (to check live whitelist membership for the specific service) +- `src/com/meesho/utilities/buTeamMapping.groovy` (for node pool selection) +- `resources/org/meesho/*.yaml` (for pod spec and image resolution) + +--- + +## Design notes + +The skill must understand the devops-lib environment mapping table precisely: + +| Branch | PR target | `cicd_environment` | +|---|---|---| +| `main`/`master`/`gcp-main` | — | `prd` | +| `develop` | — | `stg` | +| any | `main` | `int` | +| any | `develop` | `ftr` | +| `hotfix/*` | — | `prd` (sonar + tests skipped) | + +And must correctly simulate the hotfix path (sonar skipped, quality gate skipped, no CAC validation) vs the standard path. + +--- + +## Open questions + +- Should the trace show actual code line numbers for each decision, or just method names? +- For `int` and `ftr` environments where ArgoCD deploy is often skipped: should it explain why? +- Should it compare two contexts side-by-side (e.g., "what's different between develop and hotfix builds for this service")? +- Could this skill power an interactive pipeline visualiser (Mermaid diagram output)? diff --git a/docs/tribal-knowledge.md b/docs/tribal-knowledge.md new file mode 100644 index 0000000..e6e68b8 --- /dev/null +++ b/docs/tribal-knowledge.md @@ -0,0 +1,133 @@ +# Tribal Knowledge + +> Non-obvious conventions, design decisions, and operational patterns in devops-lib +> that aren't captured in comments or standard docs. Authored manually from codebase review. + +--- + +## 1. Five whitelists = five git clones per build + +`constructParam.groovy:getWhitelistedRepos` does a fresh `git clone` of `Meesho/whitelists` for every whitelist check — there is no caching between calls. A build that hits all five gates (`skip-sonar`, `app-config-disabled`, `multizone-enabled`, `allowedNonDevelopPrDeployment`, `ValidateCacConfig`) clones the repo five times. This is deliberate: each clone captures the latest whitelist state so a DevOps change takes effect on the very next build without any library release. The cost is ~5× clone latency under GitHub rate-limiting. + +**Takeaway:** Never refactor `getWhitelistedRepos` to cache the clone across calls without confirming the freshness guarantee is no longer required. + +--- + +## 2. `ringmaster-bot` is the only signal that distinguishes Ringmaster from Turbo-Turtle + +`deployRingmaster.groovy:run` switches the callback target based solely on `getCause(UserIdCause).getUserId() == "ringmaster-bot"`. There is no explicit flag or env var. If Ringmaster ever renames its bot user, callbacks silently fall through to the Turbo-Turtle endpoint, which will reject them. + +**Takeaway:** The string `"ringmaster-bot"` is a load-bearing constant. Don't change it without coordinating with the Ringmaster team. + +--- + +## 3. buildObjHelper falls through to `defaultBuild` silently + +`buildObjHelper.groovy:run` matches the `toolchain` field against a series of conditions (maven, gradle, go, node, python, php, docker). If none match, it instantiates `defaultBuild` without logging a warning. A mis-spelled toolchain value (e.g. `golang` instead of `go`) produces a silent no-op build that succeeds with no artifact. + +**Takeaway:** If a build produces no Docker image but reports success, check `toolchain` spelling in `deployment.yaml` first. + +--- + +## 4. JVM memory flags are auto-calculated — don't set them manually + +`deployArgoCD.groovy:update_helm_repo` computes `xms` and `xmx` from the pod's `memory_limit` using the formula `xms = xmx = memory_limit * 0.5`. If a service hard-codes `-Xmx` in `JAVA_OPTS`, the auto-calculated value in the Helm values will collide, with the last one winning depending on JVM arg order. + +**Takeaway:** Leave `xms`/`xmx` unset in service configs; let the pipeline compute them. If you must override, set `jvm_memory_override: true` in `deployment.yaml` to suppress auto-calc. + +--- + +## 5. Canary is mandatory for sp0 and up0 — no override + +`deployArgoCD.groovy:run` checks `priority_v2` and blocks a non-canary `prd` deploy if the priority is `sp0` or `up0`. There is no whitelist or flag to bypass this. The check happens before any Helm update, so the build fails fast. + +**Takeaway:** Any service with `priority_v2: sp0` or `up0` must have canary configured in `deployment.yaml`. Attempting a direct prd deploy will always fail at the ArgoCD stage. + +--- + +## 6. Turbo-Turtle callback uses a temp file to avoid shell escaping + +`deployRingmaster.groovy:run` passes the Turbo-Turtle JSON payload inline via `curl -d '$newCICD_JSON'`. + +**Takeaway:** If you add a new field to the Turbo-Turtle payload, add it to the temp-file write — never inline it in the curl command. + +--- + +## 7. `env.CHANGE_ID` is the canonical PR-build detector + +Every builder and stage that needs to distinguish a PR build from a branch build checks `env.CHANGE_ID` (set by the GitHub Branch Source plugin). `constructParam.groovy:run` remaps `cicd_environment` from `prd` to `int` for main/master-targeting PRs and from `stg` to `ftr` for develop-targeting PRs based on this. Don't check `env.BRANCH_NAME =~ /PR-/`; that pattern breaks on non-GitHub SCMs and on re-triggered builds. + +**Takeaway:** Use `env.CHANGE_ID` to detect PR context, not branch name patterns. + +--- + +## 8. Non-prd node pools are BU-scoped, not service-scoped + +`nodePoolSelection.groovy:run` assigns `stg`/`ftr`/`dev` pods to `{BU}-shared` pools. All services in the same BU share one node pool in non-prd environments. A memory leak or noisy-neighbour in one `supply` service degrades all other `supply` services on staging. + +**Takeaway:** Non-prd performance issues may be caused by a neighbour in the same BU pool, not the service under test. + +--- + +## 9. The `constructTemplate._construct()` method is `@NonCPS` + +`constructTemplate.groovy:_construct()` is annotated `@NonCPS` because it uses Java regex and string interpolation that is not serialisable by the Jenkins CPS engine. Any caller that invokes it inside a `parallel` block or closure must ensure the closure itself is either also `@NonCPS` or does not cross a serialisation boundary. + +**Takeaway:** Don't move `_construct` into a CPS context (e.g. by inlining its logic into a `stage` body). Keep the `@NonCPS` annotation and call it from a CPS-safe wrapper. + +--- + +## 11. Node build changes always require updating both buildNode.groovy and node-Dockerfile + +`src/com/meesho/stages/buildNode.groovy` and `resources/com/meesho/node-Dockerfile` are paired files — the Dockerfile's `else` branch (the default install command, currently `npm ci`) must stay in sync with the detection/selection logic in `buildNode.groovy`. When `npm_install_arg` is not explicitly set in `config.yaml`, `buildNode.groovy` detects the package manager at runtime and passes the install command to the Dockerfile via the `npm_install_arg` template variable. The Dockerfile's default branch handles the fallback when no explicit or detected command is provided. + +**Takeaway:** Any change to Node install logic (`npm ci`, `npm install`, pnpm detection) in `buildNode.groovy` must be paired with the same intent reflected in `node-Dockerfile`. Changing only one file leaves the two out of sync and silently breaks either the runtime path or the fallback. + +--- + +## 12. The build-tools pod image version encodes the Go and sonar-scanner toolchain + +`resources/org/meesho/prd-pod.yaml` and `stg-pod.yaml` reference a `build-tools` image from the internal GAR registry (e.g. `asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/build-tools:lunar-v2.0.21`). This image bundles the Go compiler, `sonar-scanner-cli`, and other language toolchain binaries. When a new toolchain binary is needed (e.g. a newer Go version or sonar-scanner), the fix is to bump the image tag in both pod YAMLs — not to download the tool at build time from `go.dev` or `sonarsource.com`. External downloads are a reliability risk behind a corporate network and are the wrong pattern for this repo. + +**Takeaway:** If a build stage needs a new CLI tool or toolchain binary, bump the `build-tools` image tag in `prd-pod.yaml` + `stg-pod.yaml`. Never `curl` / `wget` a tool from the internet inside a pipeline stage. + +--- + +## 13. Go sonar skip logic lives in `constructParam.groovy::skipSonarCheckForGo()` + +Quality gate and sonar skip decisions for Go builds live in `constructParam.groovy`, not inline in `buildGo.groovy`. The pattern mirrors the existing `skipSonarCheckForbidden()` method: check `getWhitelistedRepos("skip-sonar-whitelist")` and `env.BRANCH_NAME.contains("hotfix")`, return `true` if either matches. This keeps all skip-sonar policy in one place. When adding Go sonar support, add `skipSonarCheckForGo(Map config)` to `constructParam.groovy` and call it from `buildGo.groovy::buildDckr()`. + +**Takeaway:** Sonar skip logic belongs in `constructParam.groovy`. Don't embed whitelist checks or hotfix branch checks inline in language build stages. + +--- + +## 14. DinD sidecar DOCKER_HOST values must match the existing infrastructure endpoints + +When creating a Docker-in-Docker sidecar pod, set `DOCKER_HOST` in the pod YAML to the pre-provisioned TCP service endpoints — **do not use `tcp://localhost:2375`** (socket sharing). The correct values are: + +| Environment | DOCKER_HOST | +|---|---| +| prd | `dind-prd-svc` | +| stg / ftr | `dind-dev-new-svc.jenkins-new.svc.cluster.local` | + +These are exactly the same values `constructParam.groovy` sets via `accountDetails[env.cicd_environment]['dockerHost']`. Because the pod YAML and `constructParam` agree on the value, **no change to `constructParam.groovy` is needed** when adding a sidecar pod. Using `localhost` creates a conflict that requires an `env.SIDECAR_ENABLED` guard — unnecessary complexity that the TCP endpoint approach eliminates. + +Also: the DinD container image must come from the internal GAR registry: `asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/docker:28-dind`. Never use `docker:N-dind` from Docker Hub — it will be blocked by network policy and is not security-scanned. + +**Takeaway:** DinD sidecar DOCKER_HOST = `dind-prd-svc` (prd) or `dind-dev-new-svc.jenkins-new.svc.cluster.local` (stg). DinD image = internal GAR `docker:28-dind`. No constructParam changes needed. + +--- + +## 15. `scm` is unavailable inside Jenkins shared library code + +The `scm` variable (branch, remote URL, credentials) is injected by the GitHub Branch Source plugin into **consumer Jenkinsfiles** only. It is not available inside `vars/` or `src/` of the shared library. Referencing `scm.branches` or `scm.userRemoteConfigs` in library code causes a `MissingPropertyException` at runtime. + +**Takeaway:** Never access `scm` in `vars/*.groovy` or `src/com/meesho/**/*.groovy`. Use `env.BRANCH_NAME`, `env.GIT_URL`, or `env.CHANGE_*` variables instead — these are set by the plugin before library code runs. + +--- + +## 10. ArgoCD app-of-apps must be refreshed before per-service sync + +`deployArgoCD.groovy:refresh_app_of_apps` triggers a sync of the ArgoCD `app-of-apps` application before syncing the individual service app. Skipping this step means a newly created service (first deploy) won't have its ArgoCD `Application` object created yet, and the subsequent `refresh_and_sync` will target a non-existent app. + +**Takeaway:** The four-step deploy order (`update_argo_repo → refresh_app_of_apps → update_helm_repo → refresh_and_sync`) is load-bearing. Steps 2 and 4 are not interchangeable. diff --git a/docs/wiki/.citation-index.json b/docs/wiki/.citation-index.json new file mode 100644 index 0000000..a6ffc45 --- /dev/null +++ b/docs/wiki/.citation-index.json @@ -0,0 +1,39 @@ +{ + "built_against_sha": "e504a2b428e4", + "built_at": "2026-05-21T20:00:39+00:00", + "by_file": {}, + "by_page": { + "docs/wiki/pages/01-ARCHITECTURE.md": [], + "docs/wiki/pages/01-overview.md": [], + "docs/wiki/pages/02-ENTRYPOINTS.md": [], + "docs/wiki/pages/02-entry-points.md": [], + "docs/wiki/pages/03-BUILD-STAGES.md": [], + "docs/wiki/pages/03-build-dispatch.md": [], + "docs/wiki/pages/04-DEPLOY-ARGOCD.md": [], + "docs/wiki/pages/04-deploy-flow.md": [], + "docs/wiki/pages/05-ENVIRONMENT-MAPPING.md": [], + "docs/wiki/pages/05-cross-cutting.md": [], + "docs/wiki/pages/06-CONFIG-POLICY.md": [], + "docs/wiki/pages/07-LANGUAGE-BUILDS.md": [], + "docs/wiki/pages/08-DOCKERFILE-TEMPLATES.md": [], + "docs/wiki/pages/09-INFRA-PODS.md": [], + "docs/wiki/pages/10-NOTIFICATIONS.md": [], + "docs/wiki/pages/adr/adr-index.md": [], + "docs/wiki/pages/build/build-dispatch.md": [], + "docs/wiki/pages/build/config-only-detection.md": [], + "docs/wiki/pages/build/docker-tagging.md": [], + "docs/wiki/pages/build/node-paired-files.md": [], + "docs/wiki/pages/build/scm-variable-scope.md": [], + "docs/wiki/pages/concepts/observability.md": [], + "docs/wiki/pages/concepts/secrets-and-auth.md": [], + "docs/wiki/pages/concepts/whitelists.md": [], + "docs/wiki/pages/deploy/argocd-sync.md": [], + "docs/wiki/pages/deploy/ringmaster-integration.md": [], + "docs/wiki/pages/infra/node-pool-selection.md": [], + "docs/wiki/pages/policy/cac-validation.md": [], + "docs/wiki/pages/policy/multi-tenancy.md": [], + "docs/wiki/pages/policy/whitelist-system.md": [], + "docs/wiki/pages/security/security-overview.md": [] + }, + "schema_version": "v0.7.0" +} diff --git a/docs/wiki/.drift-queue/.gitkeep b/docs/wiki/.drift-queue/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/wiki/MANIFEST.md b/docs/wiki/MANIFEST.md new file mode 100644 index 0000000..d5ab9f3 --- /dev/null +++ b/docs/wiki/MANIFEST.md @@ -0,0 +1,106 @@ +# MANIFEST — devops-lib wiki + +<!-- m-wiki: manifest base-sha=e504a2b428e4 generated-at=2026-05-22 --> + +Auto-generated by `/m-wiki:wiki-init`. Each run rewrites this file in place. + +## Run summary (2026-05-22, update mode) + +| Metric | Value | +|---|---| +| Mode | update | +| base-sha | e504a2b428e4 | +| Pages on disk | 36 (10 top-level + 26 concept across `concepts/`, `build/`, `policy/`, `security/`, `deploy/`, `infra/`, `adr/`) | +| Pages newly synthesized this run | **0** — see Synthesis note below | +| Pages touched by drift refresh | 0 (B17 precheck found 0 drifted citations) | +| LINE→Symbol promotions | 0 | +| Raw inputs picked up | 0 (`raw/` empty) | +| External inputs picked up | 0 (`raw/external/` empty) | +| Reconcile inventory consumed | 0 hints across 0 pages | +| Drift-queue entries drained | 0 (empty) | +| Auto-synthesize targets folded | 0 — `docs/tribal-knowledge.md` content already represented in existing pages | +| Warnings | 3 (see below) | + +## Synthesis note (this run) + +Phase 1 step 3.6 classified `docs/tribal-knowledge.md` as `auto_synthesize` (allowlist mode is `synthesize` and the file had no `sources=[...]` provenance hit in existing pages). Phase 2 §2.6 fired the decision tree against sections §1-15 and found that **every section already has equivalent coverage in pages added by commit `4a7c4001` (2026-05-12)** and the dual-bootstrap union created by the PR #861 merge: + +| Tribal-knowledge section | Existing coverage | +|---|---| +| §1 Whitelists | `pages/concepts/whitelists.md` + `pages/policy/whitelist-system.md` | +| §2 ringmaster-bot constant | `pages/04-deploy-flow.md` + `pages/deploy/ringmaster-integration.md` | +| §3 buildObjHelper fall-through | `pages/03-build-dispatch.md` + `pages/build/build-dispatch.md` | +| §4 JVM memory formula | `pages/04-deploy-flow.md` + `pages/04-DEPLOY-ARGOCD.md` | +| §5 Canary mandatory for sp0/up0 | `pages/04-deploy-flow.md` + `pages/04-DEPLOY-ARGOCD.md` | +| §6 Turbo-Turtle inline JSON | `pages/04-deploy-flow.md` + `pages/deploy/ringmaster-integration.md` | +| §7 `env.CHANGE_ID` | `pages/05-cross-cutting.md` + `pages/05-ENVIRONMENT-MAPPING.md` | +| §8 BU-scoped node pools | `pages/05-cross-cutting.md` + `pages/infra/node-pool-selection.md` + bullet in `09-INFRA-PODS.md` | +| §9 `constructTemplate._construct()` `@NonCPS` | `pages/01-overview.md` + `pages/01-ARCHITECTURE.md` | +| §10 ArgoCD app-of-apps step order | `pages/04-deploy-flow.md` + `pages/04-DEPLOY-ARGOCD.md` | +| §11 Node build / Dockerfile pairing | `pages/build/node-paired-files.md` | +| §12 build-tools pod image versioning | bullet in `pages/09-INFRA-PODS.md` | +| §13 Go sonar skip in `constructParam` | bullets in `pages/06-CONFIG-POLICY.md` + `pages/policy/cac-validation.md` | +| §14 DinD sidecar `DOCKER_HOST` | bullet in `pages/09-INFRA-PODS.md` | +| §15 No `scm` in library code | `pages/build/scm-variable-scope.md` | + +Per the atomic-concept rule, no new pages were created — they would all duplicate existing ones. The Decision tree's "doesn't fit anywhere" branch did not fire because every section had at least one existing page covering its scope. + +I initially wrote 5 new concept pages (`node-build-dockerfile-pairing.md`, `build-tools-image.md`, `sonar-skip-routing.md`, `dind-sidecar-endpoints.md`, `no-scm-in-library.md`) before spotting the duplicates in the older topic-based hierarchy. Those 5 pages have been removed and are not in the final commit. + +## Allowlist source + +`docs/wiki/allowlist.yaml` (unchanged this run · 5 entries · last touched in PR #861) + +## Warnings + +1. **qmd registration via daemon.** The qmd HTTP MCP daemon on port 7733 serves the `devops-lib-wiki` collection (35 files indexed). `/m-wiki:wiki-search` works from a fresh Claude Code session. + +2. **`.citation-index.json` `by_file` is empty by upstream design for `.groovy` files.** `m-wiki/scripts/code_truth_precheck.py:73` defines `EXT_RE` without `groovy`. ~70 `.groovy:LINE` citations in this wiki aren't indexed. B19 pre-commit drift detection is therefore inactive for `.groovy` changes. **Upstream issue.** File on `Meesho/spells` to add `groovy` to `EXT_RE`. + +3. **Dual concept hierarchy.** This wiki currently has two parallel concept structures from independent bootstraps that got unioned in a merge: + - `pages/concepts/` (3 pages: whitelists, secrets-and-auth, observability) — added in PR #861 from `wiki: bootstrap at base-sha=5399a5ddc36b` + - `pages/{build,policy,security,deploy,infra,adr}/` (~13 pages) — added in commit `4a7c4001` from 2026-05-12 (`wiki: regenerate via /m-wiki:wiki at base-sha=28f54cf7`) + + Both structures cover overlapping concepts (e.g. whitelists are documented in both `concepts/whitelists.md` and `policy/whitelist-system.md`). qmd indexes both, so search results may return duplicates. A future cleanup PR should pick one hierarchy as canonical and graduate any unique content from the other. This run does NOT touch this — out of scope for "update mode picks up new tribal-knowledge content". + +## Pages + +``` +docs/wiki/ +├── SCHEMA.md +├── index.md +├── log.md +├── MANIFEST.md ← this file +├── allowlist.yaml +├── .citation-index.json (regenerated this run; by_file empty per warning #2) +├── .drift-queue/.gitkeep +└── pages/ + ├── 01-overview.md ┐ + ├── 01-ARCHITECTURE.md │ Dual top-level set — see warning #3 + ├── 02-entry-points.md │ + ├── 02-ENTRYPOINTS.md │ + ├── 03-build-dispatch.md │ + ├── 03-BUILD-STAGES.md │ + ├── 04-deploy-flow.md │ + ├── 04-DEPLOY-ARGOCD.md │ + ├── 05-cross-cutting.md │ + ├── 05-ENVIRONMENT-MAPPING.md┘ + ├── 06-CONFIG-POLICY.md + ├── 07-LANGUAGE-BUILDS.md + ├── 08-DOCKERFILE-TEMPLATES.md + ├── 09-INFRA-PODS.md + ├── 10-NOTIFICATIONS.md + ├── adr/adr-index.md + ├── build/ 5 pages + ├── concepts/ 3 pages (unchanged this run) + ├── deploy/ 2 pages + ├── infra/ 1 page + ├── policy/ 3 pages + └── security/ 1 page +``` + +## What this run actually did + +Net result on disk: only `.citation-index.json` regenerated + `log.md`, `MANIFEST.md`, `index.md` refreshed with the new base-sha and the dual-hierarchy note. No content additions; no content modifications outside the four metadata files. + +The user invoked `/m-wiki:wiki-init` expecting the bootstrap-deferred tribal-knowledge synthesis to fire; the script's discovery did identify the target, but the decision tree found no scope-extension work to do. Surfacing this honestly is more useful than fabricating a synthesis pass. diff --git a/docs/wiki/SCHEMA.md b/docs/wiki/SCHEMA.md new file mode 100644 index 0000000..f3b4182 --- /dev/null +++ b/docs/wiki/SCHEMA.md @@ -0,0 +1,60 @@ +# SCHEMA — devops-lib wiki + +<!-- m-wiki: schema base-sha=5399a5ddc36b generated-at=2026-05-21 --> + +LLM-maintained wiki for the **devops-lib** Jenkins shared library, generated by `/m-wiki:wiki-init` (bootstrap mode). + +## Layout + +``` +docs/wiki/ +├── SCHEMA.md ← this file (schema, layout, conventions) +├── index.md ← table of contents (auto-regenerated) +├── log.md ← per-run sync log +├── MANIFEST.md ← run statistics + page counts +├── allowlist.yaml ← qmd index allowlist (scaffolded, hand-editable) +├── .citation-index.json ← symbol-to-page mapping (auto-regenerated; v0.6+) +├── .drift-queue/ ← staged drift signals (committed, drained by next wiki-init) +└── pages/ + ├── 01-overview.md + ├── 02-entry-points.md + ├── 03-build-dispatch.md + ├── 04-deploy-flow.md + ├── 05-cross-cutting.md + └── concepts/ + ├── whitelists.md + ├── secrets-and-auth.md + └── observability.md +``` + +## Page types + +| Type | Path | Soft cap | +|---|---|---| +| `top-level` | `pages/NN-NAME.md` | narrative; no hard limit | +| `concept` | `pages/concepts/<slug>.md` | ≤600 words; lint warns >1500 | + +## Provenance line + +Every page starts with the provenance comment + a human-readable Generated line. The skill validates these on every re-run. + +``` +<!-- m-wiki: type=top-level slug=overview topic=null base-sha=5399a5ddc36b generated-at=2026-05-21 sources=[code:vars/buildPipeline.groovy, code:src/com/meesho/stages/buildObjHelper.groovy] --> + +> Generated 2026-05-21 at base-sha 5399a5ddc36b. Type: top-level. 2 sources. +``` + +## Settings + +- `max-hierarchy-depth: 1` — single-service repo, no monorepo subpaths. +- `mode: bootstrap` (this run). Subsequent runs flip to `update` once `SCHEMA.md` exists. +- `qmd-collection: devops-lib-wiki`. + +## Boundary rules + +- m-wiki **writes** only inside `docs/wiki/`. +- m-wiki **indexes** the files declared in `allowlist.yaml` (`mode: index`) and **folds** the bodies of files declared `mode: synthesize` into concept pages (on the next update run — bootstrap defers synthesis). +- `raw/` is the team's input layer; humans drop curated source docs there. None exist for this repo yet. +- `.pre-commit-config.yaml` is gitignored (per-dev); teammates must run `/m-wiki:wiki-setup` on their own clones to get drift hooks. + +See `/Users/roshan.v/.claude/plugins/marketplaces/meesho-skills/m-wiki/references/doctrine.md` for the full doctrine. diff --git a/docs/wiki/allowlist.yaml b/docs/wiki/allowlist.yaml new file mode 100644 index 0000000..b0330ca --- /dev/null +++ b/docs/wiki/allowlist.yaml @@ -0,0 +1,38 @@ +# m-wiki Index Allowlist — scaffolded by wiki-init on first run. +# Edit freely; subsequent wiki-init runs will read this file as-is. +# +# Schema: +# path — repo-relative file path or glob (no leading `/`, no `..`) +# skill — informational tag naming the skill that generates the file +# required — true → MANIFEST.md reports the file as missing if absent +# false → silently skipped when absent +# mode — synthesize: fold body into wiki pages (extends/novel/ +# contradicts/doesn't-fit decision tree, same as raw/external/). +# index: register with qmd for search only; body never read. +# Absent: legacy/undecided — wiki-init's discovery gate will +# prompt the user once per file with no provenance. +# +# Files inside docs/wiki/ MUST NOT be listed here — m-wiki indexes its +# own pages automatically and including them double-indexes. + +entries: + - path: README.md + skill: meesho-init + required: true + mode: index + - path: CLAUDE.md + skill: meesho-init + required: true + mode: index + - path: docs/architecture.md + skill: meesho-init + required: true + mode: index + - path: docs/tribal-knowledge.md + skill: pr-learnings + required: false + mode: synthesize + - path: docs/acronyms.md + skill: acronyms-docs + required: false + mode: index diff --git a/docs/wiki/index.md b/docs/wiki/index.md new file mode 100644 index 0000000..1548f8d --- /dev/null +++ b/docs/wiki/index.md @@ -0,0 +1,56 @@ +# devops-lib wiki — index + +<!-- m-wiki: index base-sha=e504a2b428e4 generated-at=2026-05-22 --> + +> Auto-regenerated by `/m-wiki:wiki-init`. Hand-editing this file is discouraged — your edits will be overwritten on the next sync. + +## Top-level pages + +| # | Page | What it covers | +|---|---|---| +| 01 | [Overview](pages/01-overview.md) | What devops-lib is, how services consume it, the load/runtime model | +| 02 | [Entry points (vars/)](pages/02-entry-points.md) | The 10 Jenkins-global steps service Jenkinsfiles call | +| 03 | [Build dispatch](pages/03-build-dispatch.md) | The `buildObjHelper` switch from `build_tool` → stage class | +| 04 | [Deploy flow](pages/04-deploy-flow.md) | The load-bearing 4-step ArgoCD deploy ceremony and the canary enforcement gate | +| 05 | [Cross-cutting patterns](pages/05-cross-cutting.md) | Retries, BU multi-tenancy, downstream auth, feature flags, parallel build | + +## Concept pages + +This wiki has two overlapping concept hierarchies — see warning #3 in [MANIFEST.md](MANIFEST.md). The `concepts/` set was added by PR #861 (my bootstrap) and the older topic-based set (`build/`, `policy/`, `security/`, `deploy/`, `infra/`) was added by `4a7c4001`. Most readers should start with whichever set comes first in their search results; both are still indexed by qmd. + +### My set (post-#861) + +| Topic | Page | +|---|---| +| Whitelists | [The five whitelist gates](pages/concepts/whitelists.md) | +| Security | [Secrets, SSH keys, hard-coded IPs](pages/concepts/secrets-and-auth.md) | +| Observability | [Logging via `log.groovy`](pages/concepts/observability.md) | + +### Older topic-based set (from `4a7c4001`) + +| Topic | Pages | +|---|---| +| Build | [build-dispatch](pages/build/build-dispatch.md) · [docker-tagging](pages/build/docker-tagging.md) · [config-only-detection](pages/build/config-only-detection.md) · [node-paired-files](pages/build/node-paired-files.md) · [scm-variable-scope](pages/build/scm-variable-scope.md) | +| Policy | [cac-validation](pages/policy/cac-validation.md) · [multi-tenancy](pages/policy/multi-tenancy.md) · [whitelist-system](pages/policy/whitelist-system.md) | +| Security | [security-overview](pages/security/security-overview.md) | +| Deploy | [argocd-sync](pages/deploy/argocd-sync.md) · [ringmaster-integration](pages/deploy/ringmaster-integration.md) | +| Infra | [node-pool-selection](pages/infra/node-pool-selection.md) | +| ADR | [adr-index](pages/adr/adr-index.md) | + +### Tribal-knowledge fold-in (this run) + +`docs/tribal-knowledge.md` sections §11-15 (added to that file via the same #861 merge) are already covered by the older topic-based set above — specifically `build/node-paired-files.md` (§11), top-level `09-INFRA-PODS.md` (§12 + §14), and `build/scm-variable-scope.md` (§15). Sections §13 (`sonar-skip-routing`) and §14 (`dind-sidecar-endpoints`) are covered as bullets in `09-INFRA-PODS.md` and `06-CONFIG-POLICY.md`. No new pages were synthesized this run — see MANIFEST for the per-section mapping. + +## Other repo docs (indexed by qmd, not part of this wiki) + +| Path | Owner skill | Role | +|---|---|---| +| [`README.md`](../../README.md) | meesho-init | Service blurb for GitHub browsers | +| [`CLAUDE.md`](../../CLAUDE.md) | meesho-init | Claude-facing repo guide | +| [`docs/architecture.md`](../architecture.md) | meesho-init | Long-form architecture | +| [`docs/acronyms.md`](../acronyms.md) | acronyms-docs | Domain abbreviations | +| [`docs/tribal-knowledge.md`](../tribal-knowledge.md) | pr-learnings | Non-obvious conventions — all sections (§1-15) already represented in existing wiki pages from earlier bootstrap commits (`4a7c4001`, `127b59a3`); no fresh fold-in this run | +| [`review-learnings.md`](../../review-learnings.md) | pr-learnings | PR-review-derived rules | +| [`BUGS_AND_IMPROVEMENTS_REPORT.md`](../../BUGS_AND_IMPROVEMENTS_REPORT.md) | (manual) | Known bug catalogue | + +Run `/m-wiki:wiki-search "<query>"` to retrieve across this wiki + the indexed files above. diff --git a/docs/wiki/log.md b/docs/wiki/log.md new file mode 100644 index 0000000..9d13e2d --- /dev/null +++ b/docs/wiki/log.md @@ -0,0 +1,6 @@ +# Sync log + +| Date (UTC) | base-sha | Mode | Pages | Externals | Citations | Raws | Warnings | Notes | +|---|---|---|---|---|---|---|---|---| +| 2026-05-21 | 5399a5ddc36b | bootstrap | 8 | 0 | 24 | 0 | 1 | First bootstrap. 1 auto-synthesize target (docs/tribal-knowledge.md) deferred per bootstrap rule — re-run wiki-init to fold it in. | +| 2026-05-22 | e504a2b428e4 | update | 36 (=) | 0 | (n/a) | 0 | 2 | No new pages this run. The B15 auto-synthesize target docs/tribal-knowledge.md (§1-15) is already fully represented in the existing wiki — the merge of PR #861 unioned two independent wiki branches (mine at base-sha 5399a5ddc36b and `4a7c4001` from 2026-05-12), and the older one already covered §11-15 via build/node-paired-files.md, build/scm-variable-scope.md, and bullets in 09-INFRA-PODS.md / 06-CONFIG-POLICY.md. Regenerated .citation-index.json (still empty by_file because `.groovy` is not in the upstream EXT_RE). Surface the dual-hierarchy state in MANIFEST warning #3 so a future cleanup PR can dedupe. | diff --git a/docs/wiki/pages/01-ARCHITECTURE.md b/docs/wiki/pages/01-ARCHITECTURE.md new file mode 100644 index 0000000..4a25f82 --- /dev/null +++ b/docs/wiki/pages/01-ARCHITECTURE.md @@ -0,0 +1,86 @@ +<!-- m-wiki: type=top-level slug=architecture topic=null base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: top-level. 0 sources. + +# Architecture + +devops-lib is a Jenkins Shared Library that implements the entire CI/CD pipeline for all Meesho microservices. Consumer services load it via `@Library('devops-lib@main')` and delegate their full build–deploy–notify lifecycle to it through a single `eksCICD(repo)` call. + +## TL;DR + +- One entry point (`vars/eksCICD.groovy`) routes to GCP (pod-based) or AWS (EKS node) infra via `CLOUD_PROVIDER`. +- A sequential flow: checkout → config parse → build stage → ArgoCD deploy → Ringmaster/Turbo-Turtle notify. +- Language-specific build logic lives in `src/com/meesho/stages/build*.groovy`; infra and policy env vars in `src/com/meesho/utilities/constructParam.groovy`. +- All deployments go through ArgoCD; no `kubectl apply` ever runs directly. +- Secrets and whitelists are injected at runtime — never hard-coded. + +## Mental model + +Think of `eksCICD` as a dispatcher: it doesn't contain any build or deploy logic itself. It (1) authenticates the triggering user, (2) selects the right infrastructure pod or node, and (3) hands off to `commonCICDFlow`, which assembles and runs the actual pipeline stages in sequence. + +Each stage is a separate Groovy class with a `run(Map config)` method. `constructParam.run()` populates `env.*` variables (registry URL, vault endpoint, ArgoCD credentials, etc.) so every downstream stage can read a consistent environment without reconfiguring itself. + +## Structure / data flow + +``` +Consumer Jenkinsfile + └─ @Library('devops-lib@main') → eksCICD(repo) + │ + ├─ [auth guard] allowedUsers check + │ + ├─ CLOUD_PROVIDER=GCP → gcpInfra() → podTemplate(INFRA_ENV-pod.yaml) → node(POD_LABEL) + │ └─ container('devops-tools') → commonCICDFlow(repo) + │ + └─ CLOUD_PROVIDER=AWS → awsInfra() → node('EKS') → commonCICDFlow(repo) + │ + ▼ + commonCICDFlow(repo) + ├─ checkOut.run(repo) ← clone service + submodules + ├─ getYamlParameter.getParam() ← parse config.yaml + ├─ buildObjHelper.run(build_tool) ← dispatch to language builder + ├─ constructParam.run(param) ← resolve env.* vars + ├─ hotFix.run(repo_name) ← skip tests/sonar on hotfix/* + └─ buildObj.run(param) ← build + deploy + notify + │ + ├─ build (Maven/Go/Node/…) + ├─ deployArgoCD.run() + └─ notify.run() → deployRingmaster.run() +``` + +## Key code locations + +| Symbol | File | What it does | +|--------|------|--------------| +| `call` | `vars/eksCICD.groovy:call` | Top-level entry — auth guard + infra routing | +| `gcpInfra` | `vars/eksCICD.groovy:gcpInfra` | Loads pod YAML from `resources/org/meesho/` and wraps in `podTemplate` | +| `awsInfra` | `vars/eksCICD.groovy:awsInfra` | Runs on static EKS node labelled `EKS` | +| `commonCICDFlow` | `vars/eksCICD.groovy:commonCICDFlow` | Orchestrates the full stage sequence | +| `run` | `src/com/meesho/stages/buildObjHelper.groovy:run` | Dispatches to language builder by `build_tool` | +| `run` | `src/com/meesho/utilities/constructParam.groovy:run` | Sets all `env.*` vars for build + deploy | +| `run` | `src/com/meesho/stages/deployArgoCD.groovy:run` | 4-step ArgoCD deploy per deployable | +| `run` | `src/com/meesho/stages/notify.groovy:run` | Slack + Ringmaster/Turbo-Turtle callback | + +## Sharp edges + +- **Authorization is strict**: only `ringmaster-bot`, `turbo-turtle`, and the `allowedUsers` list can trigger builds. Builds not in this list are rejected immediately with a message directing to Ringmaster. +- **`env.*` mutation is CPS-bound**: `constructParam.run()` sets `env.*` in a CPS method. Any utility that needs `@NonCPS` cannot read from `env.*` inside the annotation — use method parameters instead. +- **GCP vs AWS env diverge**: `constructParam.run()` has two separate `accountDetails` maps for GCP and AWS. Registry URL, bucket name, and ArgoCD cluster coordinates differ between clouds. +- **`useSidecar` flag**: services that need a sidecar container can pass `useSidecar: true` in the repo Map. This switches the pod template from `INFRA_ENV-pod.yaml` to `INFRA_ENV-sidecar-pod.yaml`. + +## Related concepts + +- [Build dispatch](build/build-dispatch.md) — how `buildObjHelper` matches `build_tool` strings +- [Environment mapping](05-ENVIRONMENT-MAPPING.md) — branch/PR → `cicd_environment` table +- [Config policy](06-CONFIG-POLICY.md) — `constructParam` env resolution + whitelist gates +- [Deploy ArgoCD](04-DEPLOY-ARGOCD.md) — 4-step deploy sequence +- [Infra pods](09-INFRA-PODS.md) — GCP pod spec selection +- [ADR index](adr/adr-index.md) — the "why" behind devops-lib's core design decisions +- [Security overview](security/security-overview.md) — trust boundaries and credential handling + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Previous](01-ARCHITECTURE.md) · [Index](../index.md) · [Next →](02-ENTRYPOINTS.md) diff --git a/docs/wiki/pages/01-overview.md b/docs/wiki/pages/01-overview.md new file mode 100644 index 0000000..423db6d --- /dev/null +++ b/docs/wiki/pages/01-overview.md @@ -0,0 +1,57 @@ +<!-- m-wiki: type=top-level slug=overview topic=null base-sha=5399a5ddc36b generated-at=2026-05-21 sources=[code:README.md, code:vars/buildPipeline.groovy, code:src/com/meesho/stages/buildObjHelper.groovy, code:resources/com/meesho/config.yaml] --> + +> Generated 2026-05-21 at base-sha 5399a5ddc36b. Type: top-level. 4 sources. + +# Overview + +**devops-lib** is the Jenkins shared library backing every Meesho service's CI/CD pipeline. Service repos import it via `@Library('devops-lib') _` at the top of their `Jenkinsfile`, then call one of the `vars/` globals (most commonly `buildPipeline { ... }` or `eksCICD { ... }`). + +It is **not** a service, has no `Dockerfile`, no `Makefile`, no test suite, no local run command. The only "runtime" is Jenkins itself: a Jenkins controller loads the library, evaluates the Jenkinsfile, and executes stages on a pod template selected by `env.INFRA_ENV`. + +## How a service repo consumes it + +```groovy +// <service-repo>/Jenkinsfile +@Library('devops-lib') _ + +buildPipeline { + repo_name = 'order-service' + build_tool = 'maven' // or 'go', 'node', 'python', 'gradle', 'docker', 'php' + maintainer = 'roshan.v' // Slack handle for #ci-cd-status mentions + skip_test = false + skip_sonar = false + push_to_jfrog = false // default-branch-only push otherwise + deployArgo = true +} +``` + +The map flows into [`vars/buildPipeline.groovy`](../../../vars/buildPipeline.groovy) which: + +1. Pins to `node('slave02')` ([line 16](../../../vars/buildPipeline.groovy)). +2. Sets `env.msg = 'Job Passed'` and wraps everything in `ansiColor` + `timestamps`. +3. Calls `checkOut` → `buildObjHelper.run(param.build_tool)` → the selected stage class's `run(param)` → `notify`. + +## What lives where + +| Path | Contains | Read by | +|---|---|---| +| [`vars/`](../../../vars/) | 10 entry-point Groovy scripts — the "public API" | Service Jenkinsfiles | +| [`src/com/meesho/stages/`](../../../src/com/meesho/stages/) | 19 stage classes (build, deploy, notify, scan) | `vars/` entry points | +| [`src/com/meesho/utilities/`](../../../src/com/meesho/utilities/) | 8 helpers — `constructParam`, `gitActions`, `nodePoolSelection`, `constructTemplate`, etc. | Stages | +| [`resources/com/meesho/`](../../../resources/com/meesho/) | Per-language Dockerfile + values.yaml + deployment.yaml; the Python validator | Stages (rendered into the service workspace) | +| [`resources/org/meesho/`](../../../resources/org/meesho/) | `{dev,stg,prd}-pod.yaml` Jenkins agent pod templates | `vars/` (via `libraryResource`) | + +## Why "no local build / no test suite" + +This is a Groovy library loaded by Jenkins, not a JVM app. There is no `build.gradle`, no `pom.xml`, no `package.json` at the repo root — Jenkins discovers `vars/` and `src/` by convention. The only way to "test" a change is to push the branch and point a Jenkins job at `@Library('devops-lib@<branch>')`. + +`BUGS_AND_IMPROVEMENTS_REPORT.md` flags the absent test suite as a P0 gap. + +## Critical conventions to know before changing anything + +- `vars/` files are the public API. Renaming or removing a global is a breaking change for every service Jenkinsfile in the org. +- The `build_tool` switch is in [`src/com/meesho/stages/buildObjHelper.groovy`](../../../src/com/meesho/stages/buildObjHelper.groovy). Unknown values fall through to `defaultBuild` silently. +- The deploy step order (`update_argo_repo` → `refresh_app_of_apps` → `update_helm_repo` → `refresh_and_sync`) is load-bearing — see [04-deploy-flow](04-deploy-flow.md). +- `constructTemplate._construct()` is `@NonCPS` ([constructTemplate.groovy:13-20](../../../src/com/meesho/utilities/constructTemplate.groovy)) — do not call it across a `parallel` boundary. + +See also: [02-entry-points](02-entry-points.md), [03-build-dispatch](03-build-dispatch.md), [04-deploy-flow](04-deploy-flow.md), [05-cross-cutting](05-cross-cutting.md), [`docs/tribal-knowledge.md`](../../tribal-knowledge.md), [`docs/acronyms.md`](../../acronyms.md). diff --git a/docs/wiki/pages/02-ENTRYPOINTS.md b/docs/wiki/pages/02-ENTRYPOINTS.md new file mode 100644 index 0000000..e9dce7a --- /dev/null +++ b/docs/wiki/pages/02-ENTRYPOINTS.md @@ -0,0 +1,72 @@ +<!-- m-wiki: type=top-level slug=entrypoints topic=null base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: top-level. 0 sources. + +# Entry Points + +The `vars/` directory contains every Groovy script that is callable from a consumer Jenkinsfile. Each file in `vars/` becomes a global function in the Jenkins pipeline namespace. + +## TL;DR + +- `eksCICD` is the primary entry point used by all microservices. +- `buildPipeline` is a legacy wrapper kept for backward compatibility. +- `gkeCICD`, `gcpMigration`, `cloudFunctionCICD` are specialised for specific infra targets. +- `createEKSconfigs` and `onlyPushtoJfrog` are utility entry points, not full CI/CD flows. +- `log`, `stageName`, `automationTest` are helper utilities exposed as global functions. + +## Mental model + +Any `.groovy` file placed in `vars/` is automatically loaded by Jenkins as a global variable/function. Consumer Jenkinsfiles call these directly: `eksCICD(repo)`, `buildPipeline(repo)`, etc. There is no package declaration in `vars/` files — they are scripts, not classes. + +## Structure / data flow + +``` +vars/ +├─ eksCICD.groovy Primary entry — all active microservices +├─ buildPipeline.groovy Legacy alias → wraps eksCICD +├─ gkeCICD.groovy GKE-specific pipeline (minimal stub) +├─ gcpMigration.groovy AWS→GCP migration helper +├─ cloudFunctionCICD.groovy Cloud Functions CI/CD +├─ createEKSconfigs.groovy EKS kubeconfig bootstrapper +├─ onlyPushtoJfrog.groovy Pushes JAR to JFrog without full pipeline +├─ automationTest.groovy Automation test runner entry +├─ buildDockerGroovyGke.groovy Docker build for GKE target +├─ log.groovy Global log.info/log.error/log.warn helpers +└─ stageName.groovy Returns stage name string for display +``` + +## Key code locations + +| Symbol | File | What it does | +|--------|------|--------------| +| `call` | `vars/eksCICD.groovy:call` | Main CI/CD flow — auth guard, infra routing | +| `call` | `vars/buildPipeline.groovy:18` | Legacy entry — delegates to eksCICD internals | +| `call` | `vars/gkeCICD.groovy:1` | GKE variant (thin stub) | +| `call` | `vars/cloudFunctionCICD.groovy:1` | Cloud Functions deployment flow | +| `call` | `vars/createEKSconfigs.groovy:1` | EKS kubeconfig bootstrap utility | +| `call` | `vars/log.groovy:1` | Global logging helper (info/error/warn) | + +## Sharp edges + +- **`vars/` scripts run in the Jenkins CPS interpreter.** Any non-serializable Java object (iterators, closures with complex state) passed from `vars/` to a `@NonCPS` method will throw `NotSerializableException` at runtime. +- **`buildPipeline` is legacy** — do not add new consumers to it. All new services should use `eksCICD`. +- **`log.groovy` shadows Jenkins' built-in `echo`** in some contexts — if you see unexpected log formatting, check whether `log.info` or `echo` was used. +- **`automationTest.groovy` is independent** — it does not go through `eksCICD`; automation test repos call it directly. +- **`scm` is only available in consumer Jenkinsfiles**: the `scm` variable (branch, remote URL, credentials) is injected by the GitHub Branch Source plugin into consumer Jenkinsfiles only — not into `vars/` or `src/` of the shared library. Referencing `scm.branches` in library code causes `MissingPropertyException` at runtime. Use `env.BRANCH_NAME`, `env.GIT_URL`, or `env.CHANGE_*` instead. + +See also: [SCM variable scope](build/scm-variable-scope.md) + +## Related concepts + +- [Architecture](01-ARCHITECTURE.md) — how eksCICD orchestrates the pipeline +- [Build stages](03-BUILD-STAGES.md) — what runs after the entry point selects infra +- [Environment mapping](05-ENVIRONMENT-MAPPING.md) — branch-to-env resolution +- [SCM variable scope](build/scm-variable-scope.md) — why `scm` is unavailable in library code + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Previous](01-ARCHITECTURE.md) · [Index](../index.md) · [Next →](03-BUILD-STAGES.md) diff --git a/docs/wiki/pages/02-entry-points.md b/docs/wiki/pages/02-entry-points.md new file mode 100644 index 0000000..7b1c1df --- /dev/null +++ b/docs/wiki/pages/02-entry-points.md @@ -0,0 +1,40 @@ +<!-- m-wiki: type=top-level slug=entry-points topic=null base-sha=5399a5ddc36b generated-at=2026-05-21 sources=[code:vars/buildPipeline.groovy, code:vars/eksCICD.groovy, code:vars/gkeCICD.groovy, code:vars/onlyPushtoJfrog.groovy, code:vars/createEKSconfigs.groovy, code:vars/cloudFunctionCICD.groovy, code:vars/log.groovy, code:vars/stageName.groovy] --> + +> Generated 2026-05-21 at base-sha 5399a5ddc36b. Type: top-level. 8 sources. + +# Entry points (vars/) + +Every file under [`vars/`](../../../vars/) is automatically exposed by Jenkins as a global step bearing the file's name (no `.groovy` suffix). Service Jenkinsfiles call them like top-level functions; renaming a file is a breaking change for every consumer. + +| File | Call signature | What it does | +|---|---|---| +| [`buildPipeline.groovy`](../../../vars/buildPipeline.groovy) | `def call(Map param)` | Legacy primary entry. Pins to `node('slave02')` ([line 16](../../../vars/buildPipeline.groovy)), runs checkout → buildObjHelper dispatch → optional automation tests → notify. | +| [`eksCICD.groovy`](../../../vars/eksCICD.groovy) | `def call(Map repo)` | Authorized-user gate ([lines 12-26](../../../vars/eksCICD.groovy) — explicit allowedUsers list), then routes GCP → pod template OR AWS → EKS node selector, then `commonCICDFlow()`. Used by most modern services. | +| [`gkeCICD.groovy`](../../../vars/gkeCICD.groovy) | `def Podcall(Map stepParams)` | Sparse legacy entry — Maven build + Docker push with hardcoded pod template. New services should not use this. | +| [`createEKSconfigs.groovy`](../../../vars/createEKSconfigs.groovy) | `def call(Map params)` | EKS Helm config generator. Validates BU/team and renders config files. Pod from `libraryResource("org/meesho/${env.INFRA_ENV}-pod.yaml")` ([line 5](../../../vars/createEKSconfigs.groovy)). | +| [`onlyPushtoJfrog.groovy`](../../../vars/onlyPushtoJfrog.groovy) | declarative pipeline (no `call()`) | One-off artifact push without the full pipeline. Choice parameter declares `['jdk8', 'jdk11', 'jdk17', 'jdk21']` ([line 16](../../../vars/onlyPushtoJfrog.groovy)) but the JAVA_HOME switch ([lines 48-53](../../../vars/onlyPushtoJfrog.groovy)) only handles `jdk17` explicitly — jdk11 and jdk21 fall through to the jdk8 default. ⚠ flagged in BUGS report. | +| [`gcpMigration.groovy`](../../../vars/gcpMigration.groovy) | `def call(Map params)` | One-off migration helper that emits a Jenkinsfile + Helm chart for a GCP-main branch. | +| [`buildDockerGroovyGke.groovy`](../../../vars/buildDockerGroovyGke.groovy) | `def run(Map config)` | Multi-module Docker build helper for GKE with artifact-version detection. | +| [`cloudFunctionCICD.groovy`](../../../vars/cloudFunctionCICD.groovy) | `def cloudFunctionCICDFlow()` | **Skeleton only.** Body is `sh 'ls -al'; echo 'Hello World'` ([lines 19-30](../../../vars/cloudFunctionCICD.groovy)) — flagged in BUGS report as not-yet-implemented. | +| [`log.groovy`](../../../vars/log.groovy) | `def info/warning/error(msg)` | ANSI-colour logging wrapper. See [concepts/observability](concepts/observability.md). | +| [`stageName.groovy`](../../../vars/stageName.groovy) | `def call(String description)` | Labels stages with environment + step counter. Reads `cicd_environment` → `INFRA_ENV` → `BUILD_ENV` → `ENVIRONMENT` (default `ftr`). | + +## The user-authorization gate (eksCICD only) + +[`eksCICD.groovy:12-26`](../../../vars/eksCICD.groovy) holds an explicit allowlist of users (`'turbo-turtle'`, `'ringmaster-bot'`, and a handful of named engineers). Builds triggered by anyone outside that list are **failed with a hard `error()`** and a banner directing them to Ringmaster. `buildPipeline` does **not** enforce this gate — services on the legacy entry point can be triggered directly. + +## Pod template wiring + +`env.INFRA_ENV` selects the Jenkins agent template via `libraryResource("org/meesho/${env.INFRA_ENV}-pod.yaml")`. Used in [`eksCICD.groovy:57`](../../../vars/eksCICD.groovy), [`createEKSconfigs.groovy:5`](../../../vars/createEKSconfigs.groovy), and [`onlyPushtoJfrog.groovy:4`](../../../vars/onlyPushtoJfrog.groovy). The resolution table: + +| `env.INFRA_ENV` | Resource | Use | +|---|---|---| +| `dev` | [`resources/org/meesho/dev-pod.yaml`](../../../resources/org/meesho/dev-pod.yaml) | Dev / sandbox | +| `stg` | [`resources/org/meesho/stg-pod.yaml`](../../../resources/org/meesho/stg-pod.yaml) | Staging | +| `prd` | [`resources/org/meesho/prd-pod.yaml`](../../../resources/org/meesho/prd-pod.yaml) | Production | + +All three are single-container pods (`devops-tools` from the `asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/build-tools:lunar-v1.0.1*` image), with `nodeSelector: {dedicated: jenkins}` and the matching toleration. + +Don't add inline `podTemplate` blocks in stage code — they bypass the central agent inventory. + +See also: [03-build-dispatch](03-build-dispatch.md), [`docs/architecture.md`](../../architecture.md). diff --git a/docs/wiki/pages/03-BUILD-STAGES.md b/docs/wiki/pages/03-BUILD-STAGES.md new file mode 100644 index 0000000..7ad1bed --- /dev/null +++ b/docs/wiki/pages/03-BUILD-STAGES.md @@ -0,0 +1,78 @@ +<!-- m-wiki: type=top-level slug=build-stages topic=null base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: top-level. 0 sources. + +# Build Stages + +Build logic lives in `src/com/meesho/stages/`. `buildObjHelper.groovy` is the dispatch router — it inspects the `build_tool` string from `config.yaml` and returns the appropriate builder class instance. + +## TL;DR + +- `buildObjHelper.run(build_tool)` returns an object with a `.run(param)` method. +- Matching uses Groovy's `switch/case` with regex patterns (e.g., `~/^go.*/`). +- Eight language builders exist: Maven, Go, Gradle, Node, Python, PHP, Rust, Docker. +- Each builder follows the same interface: `def run(Map config)`. +- Config-only change detection runs inside each builder before invoking the compiler/test runner — if only YAML files changed, the build step is skipped. + +## Mental model + +`buildObjHelper` is a factory. It reads one string (`build_tool`) and returns an object. The caller (`commonCICDFlow`) doesn't need to know which concrete class was returned — it just calls `.run(param)`. This is an interface-based dispatch pattern in Groovy without formal interfaces. + +Each builder is responsible for the complete build lifecycle of its language: compile, test, Sonar scan, Docker image build, and pushing to the registry. After building, each builder typically calls `deployArgoCD.run()` or similar to kick off the deploy phase. + +## Structure / data flow + +``` +config.yaml: build_tool: "go-1.22" + │ + ▼ +buildObjHelper.run("go-1.22") + switch "go-1.22": + case ~/^go.*/ → return new buildGo() + │ + ▼ +buildGo.buildDckr(param) + ├─ [config-only check] git diff configs/ → skip build if no src change + ├─ go build / go test + ├─ sonar scan (constructParam.skipSonarCheckForGo) + ├─ docker build + push (constructTemplate + dockerUtilities) + └─ deployArgoCD.run(...) or skip if deployArgo=false +``` + +## Key code locations + +| Symbol | File | What it does | +|--------|------|--------------| +| `run` | `src/com/meesho/stages/buildObjHelper.groovy:run` | Regex dispatch to builder class | +| `run` | `src/com/meesho/stages/buildMaven.groovy:run` | Maven/Java builder | +| `buildDckr` | `src/com/meesho/stages/buildGo.groovy:buildDckr` | Go builder | +| `buildDckr` | `src/com/meesho/stages/buildNode.groovy:buildDckr` | Node.js builder | +| `buildDckr` | `src/com/meesho/stages/buildPython.groovy:buildDckr` | Python builder | +| `run` | `src/com/meesho/stages/buildGradle.groovy:run` | Gradle/Java builder | +| `buildDckr` | `src/com/meesho/stages/buildPhp.groovy:buildDckr` | PHP builder | +| `buildDckr` | `src/com/meesho/stages/buildRust.groovy:buildDckr` | Rust builder | +| `run` | `src/com/meesho/stages/buildDocker.groovy:run` | Docker-only builder | +| `run` | `src/com/meesho/stages/hotFix.groovy:run` | Sets hot_fix flag — skips tests/sonar | +| `run` | `src/com/meesho/stages/checkOut.groovy:run` | Git checkout stage | + +## Sharp edges + +- **The dispatch is regex-based, not exact-match for most languages.** `~/^go.*/` matches `go`, `go-1.22`, `go-1.21`, etc. Typos in `config.yaml` silently fall through to `defaultBuild()` — there is no explicit error for unrecognized `build_tool` values. A mis-spelled `build_tool` produces a silent no-op build that succeeds with no artifact. +- **Go and Node entry points are `buildDckr`, not `run`**: `buildGo.groovy` and `buildNode.groovy` expose `buildDckr(Map config)` as their primary method. `buildMaven.groovy`, `buildGradle.groovy`, and `buildPython.groovy` use `run(Map config)`. This asymmetry is historical. +- **`hotFix.run()` runs before `buildObj.run()`** in `commonCICDFlow`. It sets `env.hot_fix = true` for hotfix/* branches, which causes each builder to skip tests and Sonar. +- **`buildPython` has multiple variants.** `python-3.10`, `python-3.12`, etc. all match `~/^python-.*/` and resolve to the same `buildPython` class, which reads `dockerBuildVersion` internally to select the right base image. + +## Related concepts + +- [Build dispatch](build/build-dispatch.md) — detailed regex pattern table for buildObjHelper +- [Config-only detection](build/config-only-detection.md) — how builders skip builds when only YAML changed +- [Docker tagging](build/docker-tagging.md) — `getDockerParams.getTag()` tag format +- [Dockerfile templates](08-DOCKERFILE-TEMPLATES.md) — how builder selects and renders the right Dockerfile + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Previous](02-ENTRYPOINTS.md) · [Index](../index.md) · [Next →](04-DEPLOY-ARGOCD.md) diff --git a/docs/wiki/pages/03-build-dispatch.md b/docs/wiki/pages/03-build-dispatch.md new file mode 100644 index 0000000..673cfc2 --- /dev/null +++ b/docs/wiki/pages/03-build-dispatch.md @@ -0,0 +1,66 @@ +<!-- m-wiki: type=top-level slug=build-dispatch topic=null base-sha=5399a5ddc36b generated-at=2026-05-21 sources=[code:src/com/meesho/stages/buildObjHelper.groovy, code:src/com/meesho/stages/buildMaven.groovy, code:src/com/meesho/stages/buildGo.groovy, code:src/com/meesho/stages/buildNode.groovy, code:src/com/meesho/stages/buildPython.groovy, code:src/com/meesho/stages/buildGradle.groovy] --> + +> Generated 2026-05-21 at base-sha 5399a5ddc36b. Type: top-level. 6 sources. + +# Build dispatch + +[`buildObjHelper.groovy`](../../../src/com/meesho/stages/buildObjHelper.groovy) is the central switch from the `build_tool` parameter (set in the service Jenkinsfile) to the stage class that actually runs. `vars/buildPipeline.groovy:12` calls `buildObjHelper.run(param.build_tool)` and assigns the return to `buildObj`; everything from then on flows through that object. + +## The switch (buildObjHelper.groovy:5-30) + +| `build_tool` value | Stage class | Source | +|---|---|---| +| `maven` | `buildMaven` | [`src/com/meesho/stages/buildMaven.groovy`](../../../src/com/meesho/stages/buildMaven.groovy) | +| `maven-*` (e.g. `maven-3.3-jdk-17`) | `buildMaven` | same | +| `docker` | `buildDocker` | [`src/com/meesho/stages/buildDocker.groovy`](../../../src/com/meesho/stages/buildDocker.groovy) | +| `python-*` (e.g. `python-3.10`) | `buildPython` | [`src/com/meesho/stages/buildPython.groovy`](../../../src/com/meesho/stages/buildPython.groovy) | +| `node-*` (e.g. `node-16`) | `buildNode` | [`src/com/meesho/stages/buildNode.groovy`](../../../src/com/meesho/stages/buildNode.groovy) | +| `go*` (e.g. `go1.21`) | `buildGo` | [`src/com/meesho/stages/buildGo.groovy`](../../../src/com/meesho/stages/buildGo.groovy) | +| `gradle` | `buildGradle` | [`src/com/meesho/stages/buildGradle.groovy`](../../../src/com/meesho/stages/buildGradle.groovy) | +| `php` | `buildPhp` | [`src/com/meesho/stages/buildPhp.groovy`](../../../src/com/meesho/stages/buildPhp.groovy) | +| any other value | `defaultBuild` (silent no-op) | (default case at line 28) | + +There is **no `sbt`** case and **no `rust`** case, despite what older docs may have implied (both have been reconciled out — see [`review-learnings.md`](../../../review-learnings.md) for the audit trail). + +## Per-language quirks worth knowing + +### `buildMaven` + +- CAC validation runs before build ([`buildMaven.groovy:186-201`](../../../src/com/meesho/stages/buildMaven.groovy)). +- Sonar scan triggered with `withSonarQubeEnv` ([line 245](../../../src/com/meesho/stages/buildMaven.groovy)) against `sonarqube-prd` via [`constructParam.groovy:172`](../../../src/com/meesho/utilities/constructParam.groovy). +- Quality-gate timeout: 600s on prd, 360s elsewhere ([lines 286-296](../../../src/com/meesho/stages/buildMaven.groovy)). +- JFrog push gated on branch ∈ `{master, main}` OR `push_to_jfrog=true` ([lines 377, 410](../../../src/com/meesho/stages/buildMaven.groovy)). S3 push gated on branch ∈ `{master, main, gcp-main, gcp-master}` OR `push_to_s3=true` ([lines 462, 497](../../../src/com/meesho/stages/buildMaven.groovy)). + +### `buildGo` + +- `sonar_scan()` ([lines 212-270](../../../src/com/meesho/stages/buildGo.groovy)) downloads the Go binary AND the sonar-scanner zip via `curl` inside the build stage itself ([lines 227, 236](../../../src/com/meesho/stages/buildGo.groovy)) — there is no fileExists check on `sonar-project.properties` before the scan ([line 220](../../../src/com/meesho/stages/buildGo.groovy)). +- Go test failures are caught and logged but **do not propagate** as pipeline failure ([lines 232-235](../../../src/com/meesho/stages/buildGo.groovy)): `echo "Go tests failed, but the pipeline will continue."`. +- `env.hot_fix = true` (set by [`hotFix.groovy:11`](../../../src/com/meesho/stages/hotFix.groovy)) skips Sonar AND the quality gate ([lines 25-28](../../../src/com/meesho/stages/buildGo.groovy)). +- Multi-module builds run in `parallel { }` ([lines 161-179](../../../src/com/meesho/stages/buildGo.groovy)). +- Docker push wrapped by `retryDockerPush` (5 attempts, 3s sleep) — see [05-cross-cutting](05-cross-cutting.md). + +### `buildNode` + +- `getNpmRc` ([lines 63-68](../../../src/com/meesho/stages/buildNode.groovy)) fetches `MEESHO_NPMRC_SECRET` from AWS Secrets Manager or Vault and writes it to `.npmrc` in the workspace. +- Inline `.env` writes for `GITHUB_TOKEN` and `SONAR` credentials ([lines 283-312](../../../src/com/meesho/stages/buildNode.groovy)) — note: there is **no** `truncate -s 0 .env` cleanup pattern; that was a historical PR concern that has been resolved. +- `DOCKER_BUILDKIT=0` is set explicitly ([lines 387, 392, 400, 405](../../../src/com/meesho/stages/buildNode.groovy)) — BuildKit is **disabled**, not enabled. The `BUILDKIT` acronym entry in [`docs/acronyms.md`](../../acronyms.md) was reconciled accordingly. +- Docker push via `retryDockerPush` ([line 389](../../../src/com/meesho/stages/buildNode.groovy)). + +### `buildPython` + +- Docker build with module support ([lines 89-210](../../../src/com/meesho/stages/buildPython.groovy)) — no Sonar scan (unlike Maven / Go / Node). +- Duplicate-key bug in `docker_bindings` map at [lines 75 and 78](../../../src/com/meesho/stages/buildPython.groovy) — second assignment overwrites the first. Flagged in BUGS report. + +### `buildGradle` + +- `branch_name = 'repo'` hard-coded at [lines 252 and 285](../../../src/com/meesho/stages/buildGradle.groovy) — the subsequent comparison against `'master'/'main'` therefore never matches the real branch. Flagged in BUGS report; do not assume `branch_name` is dynamic there. +- Uses `xq` to query `build.gradle` ([lines 255-256, 260, 288-289, 293](../../../src/com/meesho/stages/buildGradle.groovy)) — `xq` is an XML query tool and Gradle files are Groovy/Kotlin DSL, not XML. Flagged. + +## Adding a new build_tool + +1. Create `src/com/meesho/stages/build<Lang>.groovy` with a `run(Map param)` method. +2. Add a `case` in [`buildObjHelper.groovy`](../../../src/com/meesho/stages/buildObjHelper.groovy) — either a literal (`case 'rust'`) or a regex (`case ~/^rust-.*/`) for versioned variants. +3. Add `resources/com/meesho/<lang>-Dockerfile`, `<lang>-values.yaml`, `<lang>-deployment.yaml`. +4. There is no unit test to add — push the branch and validate via `@Library('devops-lib@<branch>')` in a sandbox service Jenkinsfile. + +See also: [04-deploy-flow](04-deploy-flow.md), [05-cross-cutting](05-cross-cutting.md), [`docs/architecture.md`](../../architecture.md). diff --git a/docs/wiki/pages/04-DEPLOY-ARGOCD.md b/docs/wiki/pages/04-DEPLOY-ARGOCD.md new file mode 100644 index 0000000..5ab5eb3 --- /dev/null +++ b/docs/wiki/pages/04-DEPLOY-ARGOCD.md @@ -0,0 +1,88 @@ +<!-- m-wiki: type=top-level slug=deploy-argocd topic=null base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: top-level. 0 sources. + +# Deploy via ArgoCD + +All Meesho microservice deployments go through ArgoCD. `deployArgoCD.groovy` orchestrates a 4-step sequence per deployable: it first commits an ArgoCD Application manifest to `devops-argo-config`, then triggers a Helm values update in `devops-helm-charts`, and finally hard-refreshes and syncs the ArgoCD application. + +## TL;DR + +- 4 steps per deployable: `update_argo_repo` → `refresh_app_of_apps` → `update_helm_repo` → `refresh_and_sync`. +- Every step commits to a separate Git repo (argo-config or helm-charts), opens a PR, merges it, and deletes the branch. +- The deploy waits for user input (checkbox UI) to choose which deployables to deploy, with a 300s timeout. +- Multizone deployables are **blocked** — they must use Ringmaster. +- JVM heap (`xms`/`xmx`) defaults to 50% of `memory_limit`, overridable via `deployment_args`. + +## Mental model + +`deployArgoCD` treats Git as the deployment API. Every config change becomes a PR in `devops-argo-config` or `devops-helm-charts`, which ArgoCD polls and syncs. The pipeline commits the changes atomically for one deployable at a time in a `for` loop. + +The user input step allows partial deploys — you can select "All" or individual apps from the checkbox list. If you skip input (timeout or `skip_user_input=true`), all apps in `deployment_order` are deployed. + +## Structure / data flow + +``` +deployArgoCD.run(repo_name, deployment_order, tag, ...) + │ + ├─ [whitelist check] constructParam.allowedNonDevelopPrDeploymentToIntRepos() + ├─ [user input] wait_for_user_input(deployment_order) — 300s timeout + │ + └─ for each deployable: + ├─ constructParam.isMultizoneEnabled(deployment) → ERROR if true + ├─ constructParam.perDeploymentVars(value_binding) ← sets env.argoURL, env.argoIncubator, etc. + │ + ├─ stage: update_argo_repo() + │ ├─ render argoApp.yaml template → devops-argo-config/applications_v2/<cluster>/<team>-<app>.yaml + │ └─ commit → PR → merge → delete branch + │ + ├─ stage: refresh_app_of_apps() + │ └─ argocd app sync <argoIncubator> + │ + ├─ stage: update_helm_repo() + │ ├─ render values.yaml template → devops-helm-charts/<env>/bu/team/app/values.yaml + │ ├─ [canary enforcement] enforce skipAnalysis=false for sp0/up0 services + │ ├─ [dependabot check] block if CRITICAL CVEs on prd sp0-sp1 deploys + │ └─ commit → PR → merge → delete branch + │ + └─ stage: refresh_and_sync() + ├─ argocd app get --hard-refresh <env>-<app_name> + └─ argocd app sync <env>-<app_name> +``` + +## Key code locations + +| Symbol | File | What it does | +|--------|------|--------------| +| `run` | `src/com/meesho/stages/deployArgoCD.groovy:run` | Main entry — user input + per-deployable loop | +| `update_argo_repo` | `src/com/meesho/stages/deployArgoCD.groovy:update_argo_repo` | Renders ArgoCD Application YAML and commits to argo-config | +| `refresh_app_of_apps` | `src/com/meesho/stages/deployArgoCD.groovy:refresh_app_of_apps` | Syncs the incubator app-of-apps | +| `update_helm_repo` | `src/com/meesho/stages/deployArgoCD.groovy:update_helm_repo` | Renders Helm values and commits to helm-charts | +| `refresh_and_sync` | `src/com/meesho/stages/deployArgoCD.groovy:refresh_and_sync` | Hard-refreshes and syncs the ArgoCD app | +| `enable_backward_compatibility` | `src/com/meesho/stages/deployArgoCD.groovy:enable_backward_compatibility` | Fills in missing deployment.yaml keys with defaults | +| `dependabotCriticalCheck` | `src/com/meesho/stages/deployArgoCD.groovy:dependabotCriticalCheck` | Blocks deploy if CRITICAL CVEs found | +| `calculate_active_processors` | `src/com/meesho/stages/deployArgoCD.groovy:calculate_active_processors` | Converts cpu_request string to JVM -XX:ActiveProcessorCount | + +## Sharp edges + +- **Branch naming is environment-derived**: `helm_branch_name` maps `main/master→main`, `develop→develop`, PR→`feature` or `pre-prod`. This is independent of `cicd_environment` — confusion between the two causes PR target mismatches. +- **Feature deployments use ingress namespacing**: for `envrn=ftr`, Helm values go into `values_properties.yaml` under `<ingress_val>/` subdirectory, and the app branch is prefixed with `<ingress_val>-`. +- **AppConfig gate**: `appConfigDisabledForbidden()` blocks stg deploys for Maven/Gradle repos that have `appConfigEnabled=false` and are not in the `app-config-disabled` whitelist. +- **xms/xmx auto-calculation**: for Maven/Gradle, heap is set to 50% of `memory_limit`. This can be overridden by `Xms<val>` or `Xmx<val>` tokens in `deployment_args`. If a service hard-codes `-Xmx` in `JAVA_OPTS`, the auto-calculated value will collide; use `jvm_memory_override: true` in `deployment.yaml` to suppress auto-calc. +- **Canary is mandatory for sp0/up0**: `deployArgoCD.groovy:run` (`src/com/meesho/stages/deployArgoCD.groovy:run`) blocks a non-canary `prd` deploy when `priority_v2` is `sp0` or `up0`. There is no whitelist or flag to bypass this check — it happens before any Helm update. +- **The 4-step order is load-bearing**: steps 2 (`refresh_app_of_apps`) and 4 (`refresh_and_sync`) are not interchangeable. Skipping step 2 on a first deploy means the ArgoCD Application object hasn't been created yet, causing step 4 to target a non-existent app. + +## Related concepts + +- [ArgoCD sync](deploy/argocd-sync.md) — detailed step-by-step sync sequence +- [Ringmaster integration](deploy/ringmaster-integration.md) — when deployRingmaster runs instead +- [Whitelist system](policy/whitelist-system.md) — multizone + allowedNonDevelop gates +- [Node pool selection](infra/node-pool-selection.md) — how nodeSelectorValue is computed + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Previous](03-BUILD-STAGES.md) · [Index](../index.md) · [Next →](05-ENVIRONMENT-MAPPING.md) diff --git a/docs/wiki/pages/04-deploy-flow.md b/docs/wiki/pages/04-deploy-flow.md new file mode 100644 index 0000000..6aa1b08 --- /dev/null +++ b/docs/wiki/pages/04-deploy-flow.md @@ -0,0 +1,87 @@ +<!-- m-wiki: type=top-level slug=deploy-flow topic=null base-sha=5399a5ddc36b generated-at=2026-05-21 sources=[code:src/com/meesho/stages/deployArgoCD.groovy, code:src/com/meesho/stages/deployRingmaster.groovy, code:src/com/meesho/stages/notify.groovy, code:src/com/meesho/stages/helmGenerator.groovy] --> + +> Generated 2026-05-21 at base-sha 5399a5ddc36b. Type: top-level. 4 sources. + +# Deploy flow + +The deploy phase is owned by [`deployArgoCD.groovy`](../../../src/com/meesho/stages/deployArgoCD.groovy) (the ArgoCD ceremony) and [`deployRingmaster.groovy`](../../../src/com/meesho/stages/deployRingmaster.groovy) (the callback to the higher-level deploy controllers). Two things to never mess with: the four-step order, and the canary gate. + +## The four-step ArgoCD ceremony — load-bearing + +`deployArgoCD.run()` ([lines 73-107](../../../src/com/meesho/stages/deployArgoCD.groovy)) calls these in strict order: + +``` +1. update_argo_repo (line 91) pushes the ArgoApplication YAML into devops-argo-config + so the app-of-apps registry sees the new app. +2. refresh_app_of_apps (line 94) argocd app sync ${appofapps} — materialises the new + Application object before the per-service sync needs it. +3. update_helm_repo (line 97) computes xms/xmx, renders values.yaml, pushes into + devops-helm-charts. +4. refresh_and_sync (line 100) argocd app sync ${app_name} --hard-refresh — the + actual service rollout. Uses --http-retry-max 3 + --retry-backoff-duration 1m (lines 506, 526). +``` + +Steps 2 and 4 are **not interchangeable**. On first-deploy of a brand-new service, the Application object doesn't exist yet — step 2 creates it (as a downstream effect of the app-of-apps sync), step 4 reads it. Swap the order and step 4 fails on a missing Application. See [`docs/tribal-knowledge.md`](../../tribal-knowledge.md) §10 for the post-incident note. + +## JVM memory auto-calculation + +`update_helm_repo` ([lines 220-255](../../../src/com/meesho/stages/deployArgoCD.groovy)) derives JVM memory flags from the pod's `memory_limit`: + +```groovy +memory_value = memory_limit * 0.5 +xms = "${memory_value}M" +xmx = "${memory_value}M" +``` + +Both flags are set equal, derived from `memory_limit` (not `memory_request`), with no 0.75 multiplier and no 64m rounding. The historical claim "`xmx = memory_request * 0.75, xms = xmx * 0.5`" was wrong and has been reconciled out of [`docs/tribal-knowledge.md`](../../tribal-knowledge.md). + +**Do not hard-code `-Xmx` in `JAVA_OPTS`** — the auto-computed value will collide with it, and the last value seen by the JVM wins depending on arg order. There is no `jvm_memory_override` flag (that referenced flag does not exist in code). + +## Canary enforcement — sp0 / up0 in prd + +[`deployArgoCD.groovy:408-430`](../../../src/com/meesho/stages/deployArgoCD.groovy): + +``` +enforceCanary = (priority_v2 ∈ {sp0, up0}) + ∧ (envrn == 'prd') + ∧ ¬(service is canary | cron | worker | scheduler | consumer | node | headless) +``` + +When `enforceCanary` is true, the deploy hard-errors unless the Helm values declare: + +- `canary.enabled = true` +- `canary.skipAnalysis = false` +- `canary.enableManualPromotion = true` + +There is **no whitelist** and **no bypass flag**. A service that needs to skip canary on a high-priority prd path has to either (a) be classified out of `sp0/up0`, or (b) match one of the exempted service types listed above. + +## Ringmaster vs Turbo-Turtle routing + +[`deployRingmaster.groovy:55-89`](../../../src/com/meesho/stages/deployRingmaster.groovy): + +```groovy +def build_user = currentBuild.rawBuild.getCause(Cause.UserIdCause).getUserId() +if (build_user == "ringmaster-bot") { + callApi(url, header, jsonData) // → ringmaster endpoint +} else { + // → http://turbo-turtle.meeshogcp.in (line 71) + // http://turbo-turtle.admin.meeshogcp.in (line 74) + sh "curl -s -X POST -H '$newCICD_Header' -w '\\n%{response_code}' $newCICD_URL -d '$newCICD_JSON'" +} +``` + +The string `"ringmaster-bot"` is a **load-bearing constant**. Renaming the bot user silently routes every Ringmaster callback to Turbo-Turtle, which rejects them. + +The JSON payload is passed **inline** via `-d '$newCICD_JSON'` ([line 80](../../../src/com/meesho/stages/deployRingmaster.groovy)). Earlier documentation claimed a temp-file + `curl -d @<file>` pattern with `finally`-block cleanup — that pattern does **not** exist in the current code and was reconciled out of `docs/tribal-knowledge.md`. + +## Deployment-tracker callback (notify.groovy) + +[`notify.groovy:108-152`](../../../src/com/meesho/stages/notify.groovy) POSTs to one of: + +- `http://deployment-tracker.meeshoint.in/api/1.0/deployment-tracker/jenkins/create` (AWS) +- `http://deployment-tracker.prd.meesho.int/api/1.0/deployment-tracker/jenkins/create` (GCP) + +Only triggered on branches `main` / `master` / `gcp-main` / `gcp-master` ([line 34](../../../src/com/meesho/stages/notify.groovy)). Payload includes repository, team, link, job_name, tag, status, commit_id, error_msg. No retry — a single failed POST means the dashboard miscounts that deploy. + +See also: [05-cross-cutting](05-cross-cutting.md), [concepts/whitelists](concepts/whitelists.md), [`docs/tribal-knowledge.md`](../../tribal-knowledge.md). diff --git a/docs/wiki/pages/05-ENVIRONMENT-MAPPING.md b/docs/wiki/pages/05-ENVIRONMENT-MAPPING.md new file mode 100644 index 0000000..f60ffd3 --- /dev/null +++ b/docs/wiki/pages/05-ENVIRONMENT-MAPPING.md @@ -0,0 +1,70 @@ +<!-- m-wiki: type=top-level slug=environment-mapping topic=null base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: top-level. 0 sources. + +# Environment Mapping + +`cicd_environment` is the single string that controls which registry, vault, ArgoCD cluster, and GCS bucket the pipeline uses. It is derived from the branch name (for push builds) or the PR target branch (for PR builds). + +## TL;DR + +- Push to `main`/`master`/`gcp-main` → `prd`. +- Push to `develop` → `stg`. +- PR targeting `main` → `int` (pre-prod). +- PR targeting `develop` → `ftr` (feature). +- Any other branch → `ftr` (fallback). +- Hotfix branches skip tests and Sonar but still map to `prd`. +- `env.CHANGE_ID` being set signals a PR build and flips the mapping table. + +## Mental model + +`constructParam.run()` switches on `env.CHANGE_ID` to decide which `environment_map` to use. Without `CHANGE_ID` (push build), branches map to their canonical environments. With `CHANGE_ID` (PR build), the same branch names map to the pre-prod equivalents: `main→int`, `develop→ftr`. This double-mapping is why `develop` can be either `stg` (push) or `ftr` (PR). + +## Structure / data flow + +``` +env.CHANGE_ID not set (push build): + environment_map = { + master/main/gcp-main/farmiso-main/gcp-master → prd + develop/gcp-dev → stg + } + branch not in map → ftr (default) + +env.CHANGE_ID set (PR build): + branch_name = env.CHANGE_TARGET + environment_map = { + master/main/gcp-main/farmiso-main/gcp-master → int + develop/gcp-dev → ftr + } + +env.cicd_environment = environment_map[branch_name] +``` + +## Key code locations + +| Symbol | File | What it does | +|--------|------|--------------| +| `run` | `src/com/meesho/utilities/constructParam.groovy:run` | Sets `env.cicd_environment` + all env vars | +| `run` | `src/com/meesho/stages/deployArgoCD.groovy:run` | Has its own `branch_param_map` for helm/argo branch naming | +| `run` | `src/com/meesho/stages/hotFix.groovy:run` | Detects `hotfix/*` branch and sets `env.hot_fix=true` | + +## Sharp edges + +- **Two separate maps exist**: `constructParam` has one map for `cicd_environment`; `deployArgoCD` has a different `branch_param_map` for the Helm/Argo Git branch names (`main`, `develop`, `feature`, `pre-prod`). These are not the same and must not be conflated. +- **`gcp-dev` maps to `stg`**: services on the `gcp-dev` branch deploy to staging, not a separate dev environment. +- **Toolchain override**: if `env.INFRA_ENV == 'toolchain'` and the service is Node, `branch_name` is forced to `develop` so the image goes to `stg` toolchain registry regardless of actual branch. +- **`farmiso-main`** is treated as an alias for `main` — it maps to `prd` on both push and PR builds to `main`. +- **Use `env.CHANGE_ID` to detect PR context — not `env.BRANCH_NAME =~ /PR-/`**: the `BRANCH_NAME =~ /PR-/` pattern breaks on non-GitHub SCMs and re-triggered builds. `env.CHANGE_ID` is the canonical PR-build detector set by the GitHub Branch Source plugin and is used throughout `constructParam.groovy:run` (`src/com/meesho/utilities/constructParam.groovy:run`). + +## Related concepts + +- [Config policy](06-CONFIG-POLICY.md) — `constructParam` sets env vars after mapping +- [Architecture](01-ARCHITECTURE.md) — where `commonCICDFlow` calls `constructParam` + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Previous](04-DEPLOY-ARGOCD.md) · [Index](../index.md) · [Next →](06-CONFIG-POLICY.md) diff --git a/docs/wiki/pages/05-cross-cutting.md b/docs/wiki/pages/05-cross-cutting.md new file mode 100644 index 0000000..869eb1b --- /dev/null +++ b/docs/wiki/pages/05-cross-cutting.md @@ -0,0 +1,84 @@ +<!-- m-wiki: type=top-level slug=cross-cutting topic=null base-sha=5399a5ddc36b generated-at=2026-05-21 sources=[code:src/com/meesho/utilities/constructParam.groovy, code:src/com/meesho/utilities/nodePoolSelection.groovy, code:src/com/meesho/utilities/gitActions.groovy, code:src/com/meesho/stages/deployArgoCD.groovy, code:src/com/meesho/stages/hotFix.groovy, code:vars/log.groovy] --> + +> Generated 2026-05-21 at base-sha 5399a5ddc36b. Type: top-level. 6 sources. + +# Cross-cutting patterns + +## Retries + +| Caller | Downstream | Retry | +|---|---|---| +| [`deployArgoCD.groovy:506,526`](../../../src/com/meesho/stages/deployArgoCD.groovy) | ArgoCD sync | `argocd app sync --http-retry-max 3 --retry-backoff-duration 1m` | +| [`buildNode.groovy:389`](../../../src/com/meesho/stages/buildNode.groovy), [`buildGo.groovy:172`](../../../src/com/meesho/stages/buildGo.groovy) | Docker registry push | `retryDockerPush` — 5 attempts with 3s sleep between | +| `buildMaven`, `buildGradle`, `buildPython`, `buildPhp` | various | **No retry** | +| [`securityScan.groovy:12`](../../../src/com/meesho/stages/securityScan.groovy) | In-house scanner | **No retry** | +| [`notify.groovy:108-152`](../../../src/com/meesho/stages/notify.groovy) | Deployment Tracker | **No retry** — single failed POST = missed deploy in the dashboard | + +`BUGS_AND_IMPROVEMENTS_REPORT.md` lists "no retry mechanism" as P1; the reconciled-in nuance is that retry IS present for ArgoCD sync + Docker push, but not for most other downstream calls. + +## BU multi-tenancy + +`env.BU` and `config.bu` flow from CAC config (read once by [`constructParam.groovy`](../../../src/com/meesho/utilities/constructParam.groovy)) and drive: + +| Use | Code site | +|---|---| +| ArgoCD namespace | `argocd-${env.BU}-prd` — [`constructParam.groovy:318`](../../../src/com/meesho/utilities/constructParam.groovy) | +| GCP cluster name | `k8s-${env.BU}-prd-ase1` — [`constructParam.groovy:308`](../../../src/com/meesho/utilities/constructParam.groovy) | +| GCP project | `meesho-${config.bu}-prd-0622` — [`constructParam.groovy:178`](../../../src/com/meesho/utilities/constructParam.groovy) | +| Helm chart path | `${env.helmChartsPath}/${config.bu}/...` — [`helmGenerator.groovy:150`](../../../src/com/meesho/stages/helmGenerator.groovy), [`deployArgoCD.groovy:293`](../../../src/com/meesho/stages/deployArgoCD.groovy) | +| Non-prd node pool | `${env.BU}-shared` (all `dev`/`ftr`/`stg` traffic for the BU collapses to one pool) — [`nodePoolSelection.groovy:133-135`](../../../src/com/meesho/utilities/nodePoolSelection.groovy) | + +**Implication for non-prd capacity planning:** a single noisy service in `bu=supply` degrades every other `supply` service on staging, because they all share `supply-shared`. See [`docs/tribal-knowledge.md`](../../tribal-knowledge.md) §8. + +## Feature flags / toggle conditions + +| Flag | Set by | Effect | Read at | +|---|---|---|---| +| `env.hot_fix` | [`hotFix.groovy:11`](../../../src/com/meesho/stages/hotFix.groovy) | Skips Sonar + quality gate; sets canary `skipAnalysis` | [`buildGo.groovy:25-28`](../../../src/com/meesho/stages/buildGo.groovy), [`buildMaven.groovy:37-40`](../../../src/com/meesho/stages/buildMaven.groovy), [`deployArgoCD.groovy:355`](../../../src/com/meesho/stages/deployArgoCD.groovy) | +| `config.skip_sonar` | Service Jenkinsfile param | Skips quality-gate check (gated by whitelist for Maven prd) | [`buildMaven.groovy:18,240-243`](../../../src/com/meesho/stages/buildMaven.groovy), [`constructParam.groovy:51-56`](../../../src/com/meesho/utilities/constructParam.groovy) | +| `config.skip_security_scan` | Service Jenkinsfile param | Skips POST to security-scan endpoint | [`securityScan.groovy:7-9`](../../../src/com/meesho/stages/securityScan.groovy) | +| `config.push_to_jfrog` | Service Jenkinsfile param | Allow non-default-branch JFrog push (default branches: master, main) | [`buildMaven.groovy:19,377`](../../../src/com/meesho/stages/buildMaven.groovy) | +| `config.push_to_s3` | Service Jenkinsfile param | Allow non-default-branch S3 push (default branches: master, main, gcp-main, gcp-master) | [`buildMaven.groovy:20,462`](../../../src/com/meesho/stages/buildMaven.groovy) | +| `env.INFRA_ENV == 'toolchain'` | CAC config | Skips Docker push, uses latest-tag logic | [`buildGo.groovy:40-58`](../../../src/com/meesho/stages/buildGo.groovy), [`buildNode.groovy:215-239`](../../../src/com/meesho/stages/buildNode.groovy) | +| `env.CHANGE_ID` | GitHub Branch Source plugin | PR-build detection; remaps `cicd_environment` (`prd→int` for main/master PRs, `stg→ftr` for develop PRs) | [`constructParam.groovy:107-110`](../../../src/com/meesho/utilities/constructParam.groovy) | + +**Do not use `env.BRANCH_NAME =~ /PR-/`** as a PR-build detector — it breaks on re-triggered builds and non-GitHub SCMs. + +## Parallel execution + +The pipeline is **almost entirely linear**. The one confirmed `parallel { }` block is in multi-module Go builds at [`buildGo.groovy:161-179`](../../../src/com/meesho/stages/buildGo.groovy): + +```groovy +for (m in modules) { + moduleBuilds["build-${moduleName}"] = { ... } +} +parallel moduleBuilds +``` + +Anything called from inside that closure must respect the `@NonCPS` rule (see [`constructTemplate.groovy:13-20`](../../../src/com/meesho/utilities/constructTemplate.groovy) — the template engine wrapper is `@NonCPS` because `SimpleTemplateEngine` is non-serialisable). + +## Exception handling — inconsistent on purpose-ish + +`catch (Exception e)` appears ~87 times across stages. Three observed patterns: + +| Pattern | Example | Behaviour | +|---|---|---| +| Catch + log + **rethrow** | [`buildNode.groovy:20-26`](../../../src/com/meesho/stages/buildNode.groovy) | Hard-fail the stage. Most common. | +| Catch + log + **swallow** (loop body) | [`deployArgoCD.groovy:103-106`](../../../src/com/meesho/stages/deployArgoCD.groovy) | Continue with the next deployment in the loop. Intentional for multi-deploy resilience. | +| Catch + log + **swallow** (silent fall-through) | [`buildNode.groovy:199-200`](../../../src/com/meesho/stages/buildNode.groovy) | Recovers with a sensible default. Use with caution — readers don't always realise the stage "succeeded" while masking a real error. | + +When adding new error handling, prefer the rethrow pattern unless the loop-resilience semantics are an explicit requirement. + +## Validation: `validate_configs.py` + +[`resources/com/meesho/validate_configs.py`](../../../resources/com/meesho/validate_configs.py) (1207 lines) is the monolithic CAC schema validator, invoked from Groovy via `sh`. Key validation classes: + +- **Schema** ([line 949](../../../resources/com/meesho/validate_configs.py)) — `application-{dev,int,stg,prd}.yml` against `application-schema.yml`. +- **Cross-env endpoint check** ([line 1010](../../../resources/com/meesho/validate_configs.py)) — prevents stg configs from referencing prd endpoints. +- **Secrets detection** ([line 1099](../../../resources/com/meesho/validate_configs.py)) — via `detect_secrets` library. +- **DB URL patterns** ([lines 41-59](../../../resources/com/meesho/validate_configs.py)) — PostgreSQL/MySQL/MongoDB/Redis/SQLite URI shapes. +- **Zookeeper endpoint patterns** — env-specific allow-lists. + +`validate_configs_v2.py` (1261 lines) is the eventual replacement; both are referenced today. + +See also: [concepts/whitelists](concepts/whitelists.md), [concepts/secrets-and-auth](concepts/secrets-and-auth.md), [concepts/observability](concepts/observability.md). diff --git a/docs/wiki/pages/06-CONFIG-POLICY.md b/docs/wiki/pages/06-CONFIG-POLICY.md new file mode 100644 index 0000000..c76fa40 --- /dev/null +++ b/docs/wiki/pages/06-CONFIG-POLICY.md @@ -0,0 +1,82 @@ +<!-- m-wiki: type=top-level slug=config-policy topic=null base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: top-level. 0 sources. + +# Config and Policy + +`constructParam.groovy` does two things: it resolves all `env.*` variables from `config.yaml` and the current branch/environment, and it enforces five policy gates by fetching `Meesho/whitelists` at runtime. + +## TL;DR + +- `config.yaml` is the service's contract — `getYamlParameter.getParam()` reads it before any build runs. +- `constructParam.run()` sets 15+ `env.*` vars (registry, vault, sonar, GCPProject, etc.) based on `cicd_environment`. +- Five whitelist files are fetched from `Meesho/whitelists` at runtime via `git clone` — not bundled. +- Whitelisted policies: skip-sonar, app-config-disabled, multizone, allowedNonDevelopPrToInt, ValidateCacConfig. +- `ValidateCacConfig` gate triggers `validate_configs_v2.py` on Go and Maven PRs. + +## Mental model + +`constructParam` is the policy layer. All build and deploy stages read `env.*` but never set it — they are consumers. `constructParam` is the sole producer. This ensures every stage shares a consistent view of which cloud, environment, registry, and policy applies to this build. + +The whitelist pattern allows DevOps to grant exceptions without modifying any service's code — a repo is added to `Meesho/whitelists/skip-sonar-whitelist.yaml` and the next build automatically picks up the exception. + +## Structure / data flow + +``` +config.yaml (in service repo): + repo_name, build_tool, dockerBuildVersion, bu, team, + deployment_order, notify_channel, skip_sonar, deployArgo, appConfigEnabled + +getYamlParameter.getParam(repo_name) → param Map + +constructParam.run(param): + ├─ environment_map → env.cicd_environment + ├─ GCP accountDetails[env.cicd_environment]: + │ env.GCPProject, env.registry, env.buildRegistry + │ env.vaultURL/Token, env.sonarURL/Token + │ env.objBucket, env.DOCKER_HOST + │ + ├─ [whitelist gate] skipSonarCheckForbidden() → clone Meesho/whitelists/skip-sonar-whitelist.yaml + ├─ [whitelist gate] appConfigDisabledForbidden() → app-config-disabled.yaml + ├─ [whitelist gate] isMultizoneEnabled() → multizone-enabled-repos.yaml + ├─ [whitelist gate] allowedNonDevelopPrDeploymentToIntRepos() → allowedNonDevelopPrDeploymentToInt.yaml + └─ [whitelist gate] ValidateCacConfigForRepo() → ValidateCacConfig.yaml +``` + +## Key code locations + +| Symbol | File | What it does | +|--------|------|--------------| +| `run` | `src/com/meesho/utilities/constructParam.groovy:run` | Main env var setter | +| `perDeploymentVars` | `src/com/meesho/utilities/constructParam.groovy:perDeploymentVars` | Sets per-deployable ArgoCD vars | +| `getWhitelistedRepos` | `src/com/meesho/utilities/constructParam.groovy:getWhitelistedRepos` | Clones Meesho/whitelists and reads a YAML file | +| `skipSonarCheckForbidden` | `src/com/meesho/utilities/constructParam.groovy:skipSonarCheckForbidden` | Blocks Maven prd builds with skip_sonar=true if not whitelisted | +| `appConfigDisabledForbidden` | `src/com/meesho/utilities/constructParam.groovy:appConfigDisabledForbidden` | Blocks stg deploys if appConfig disabled and not whitelisted | +| `isMultizoneEnabled` | `src/com/meesho/utilities/constructParam.groovy:isMultizoneEnabled` | Returns true for deployables in multizone whitelist | +| `ValidateCacConfigForRepo` | `src/com/meesho/utilities/constructParam.groovy:ValidateCacConfigForRepo` | Returns true if repo must run CAC validation | +| `getParam` | `src/com/meesho/utilities/getYamlParameter.groovy:getParam` | Reads a YAML file from the workspace | + +## Sharp edges + +- **Each whitelist call does a fresh `git clone`**: `getWhitelistedRepos()` clones `Meesho/whitelists` into a `whitelist/` subdirectory every time it's called. Five separate calls = five clones in the same build. Network latency here directly adds to build time. This is deliberate — each clone captures the latest whitelist state so a DevOps policy change takes effect on the very next build without a library release. Never cache across calls. +- **Go sonar-skip logic is in `constructParam.groovy`, not `buildGo.groovy`**: `skipSonarCheckForGo(Map config)` (`src/com/meesho/utilities/constructParam.groovy:skipSonarCheckForGo`) centralises all skip-sonar policy. Embedding whitelist checks inline in language build stages is the wrong pattern. +- **`perDeploymentVars` must run before ArgoCD steps**: it sets `env.argoURL`, `env.argoCreds`, `env.argoAppNS`, and `env.argoIncubator` per deployable. Calling ArgoCD stages before this results in empty ArgoCD credentials. +- **`bu` drives GCP project name**: `prodGCPProject = "meesho-${config.bu}-prd-0622"`. An invalid or misspelled `bu` in `config.yaml` produces a nonexistent GCP project name. +- **Toolchain env skips several policies**: when `env.INFRA_ENV == 'toolchain'`, skip_sonar is forced true and vault/sonar are pointed at `toolchain-dind-dev-svc`. + +## Related concepts + +- [Whitelist system](policy/whitelist-system.md) — detailed whitelist file inventory +- [CAC validation](policy/cac-validation.md) — what happens when ValidateCacConfig=true +- [Multi-tenancy](policy/multi-tenancy.md) — BU/team mapping and initials +- [Environment mapping](05-ENVIRONMENT-MAPPING.md) — how cicd_environment is determined +- [Security overview](security/security-overview.md) — credential handling, trust boundaries, security rules for new code +- [ADR index](adr/adr-index.md) — architectural decisions behind the whitelist and policy model (ADR-0003, ADR-0004) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Previous](05-ENVIRONMENT-MAPPING.md) · [Index](../index.md) · [Next →](07-LANGUAGE-BUILDS.md) diff --git a/docs/wiki/pages/07-LANGUAGE-BUILDS.md b/docs/wiki/pages/07-LANGUAGE-BUILDS.md new file mode 100644 index 0000000..96387ed --- /dev/null +++ b/docs/wiki/pages/07-LANGUAGE-BUILDS.md @@ -0,0 +1,87 @@ +<!-- m-wiki: type=top-level slug=language-builds topic=null base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: top-level. 0 sources. + +# Language Builds + +Each language builder in `src/com/meesho/stages/` implements a `run(Map config)` method. They share a common structure: read config → config-only check → compile/test → Sonar → Docker build → push → deploy. + +## TL;DR + +- **Maven** (`buildMaven`): Java services using pom.xml. Handles multi-module layouts. Skips tests/sonar on hotfix. Pushes JAR to JFrog optionally. +- **Go** (`buildGo`): Uses Athens proxy (`env.goProxyUrl`) for module caching. Sonar skipped if whitelisted or toolchain env. +- **Node** (`buildNode`): Multiple variants (`node-18`, `node-20`, etc.). Toolchain-aware image path includes `TOOLCHAIN_ENV` segment. +- **Python** (`buildPython`): `python-3.10`, `python-3.12`, etc. — version in `dockerBuildVersion`, not separate files. +- **Gradle** (`buildGradle`): Android/Kotlin server builds. +- **PHP** (`buildPhp`): PHP services, uses `php-values.yaml` Helm template. +- **Rust** (`buildRust`): Rust services. +- **Docker** (`buildDocker`): Docker-only builds — no compiler, just image assembly. + +## Mental model + +Every builder reads `config.build_tool` (via `buildObjHelper` dispatch) and `config.dockerBuildVersion`. The `dockerBuildVersion` controls which Dockerfile template is selected from `resources/com/meesho/` — the template name roughly mirrors the version string (e.g., `maven-21`, `go-1.22`, `node-20`). + +Config-only change detection is embedded in each builder. If `git diff HEAD~1 -- configs/` shows only YAML changes, the builder skips compilation and fetches the last image tag from GAR instead. + +## Structure / data flow + +``` +buildMaven.run(config): + env.TAG = "v${pom-version}" or hotfix → "v${version}-HOT" + ├─ [config-only] check s3/GAR for existing artifact + ├─ stage: Test → mvn test -P${java_version} + ├─ stage: Sonar → mvn sonar:sonar (skip if skipSonarCheckForbidden) + ├─ stage: Docker Build → constructTemplate.renderTemplate(binding, 'maven-Dockerfile') + ├─ stage: Docker Push → dockerUtilities.buildAndPush(...) + └─ deployArgoCD.run(...) or deployJar.run(...) + +buildGo.buildDckr(config): + env.TAG = getDockerParams.getTag(repo_name) + ├─ [config-only] skip build if only configs/ changed + ├─ stage: Build/Test → go build, go test (with Athens proxy) + ├─ stage: Sonar → sonar-scanner (skip if skipSonarCheckForGo) + ├─ stage: Docker Build → constructTemplate.renderTemplate(binding, 'go-Dockerfile') + └─ deployArgoCD.run(...) + +buildNode.buildDckr(config): + env.TAG = getDockerParams.getTag(repo_name) + ├─ npm install + npm run build + ├─ stage: Sonar → sonar-scanner (node sonar/sonar-scanner.ts) + ├─ stage: Docker Build → constructTemplate.renderTemplate(binding, 'node-Dockerfile') + └─ deployArgoCD.run(...) +``` + +## Key code locations + +| Symbol | File | What it does | +|--------|------|--------------| +| `run` | `src/com/meesho/stages/buildMaven.groovy:run` | Maven build lifecycle | +| `buildDckr` | `src/com/meesho/stages/buildGo.groovy:buildDckr` | Go build lifecycle | +| `buildDckr` | `src/com/meesho/stages/buildNode.groovy:buildDckr` | Node build lifecycle | +| `buildDckr` | `src/com/meesho/stages/buildPython.groovy:buildDckr` | Python build lifecycle | +| `run` | `src/com/meesho/stages/buildGradle.groovy:run` | Gradle build lifecycle | +| `buildDckr` | `src/com/meesho/stages/buildRust.groovy:buildDckr` | Rust build lifecycle | +| `retryDockerPush` | `src/com/meesho/utilities/dockerUtilities.groovy:retryDockerPush` | Docker push with retry | +| `getTag` | `src/com/meesho/utilities/getDockerParams.groovy:getTag` | Image tag: v{version}-{sha}-{epoch} | + +## Sharp edges + +- **`python-3.10.12` is aliased to `python-3.7` in Helm values**: `deployArgoCD.update_helm_repo` remaps `python-3.10.12` → `python-3.7` for the `dockerBuildVersion` key passed to the Helm template. Other python versions pass through as-is. +- **Maven multi-module builds**: `getDockerParams.getModules()` reads `<modules>` from `pom.xml`. If modules exist, a Docker image is built per module. +- **toolchain env Node builds** include `TOOLCHAIN_ENV` in the image path: `${cicd_environment}/${TOOLCHAIN_ENV}/${team}/${repo}`. Standard builds omit the `TOOLCHAIN_ENV` segment. +- **`skip_s3_check=true` forces a fresh build** even if an artifact already exists for the same commit. + +## Related concepts + +- [Build dispatch](build/build-dispatch.md) — regex routing to the correct builder +- [Config-only detection](build/config-only-detection.md) — when builds are skipped +- [Dockerfile templates](08-DOCKERFILE-TEMPLATES.md) — template rendering +- [Docker tagging](build/docker-tagging.md) — tag format used by all builders + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Previous](06-CONFIG-POLICY.md) · [Index](../index.md) · [Next →](08-DOCKERFILE-TEMPLATES.md) diff --git a/docs/wiki/pages/08-DOCKERFILE-TEMPLATES.md b/docs/wiki/pages/08-DOCKERFILE-TEMPLATES.md new file mode 100644 index 0000000..9ae31d6 --- /dev/null +++ b/docs/wiki/pages/08-DOCKERFILE-TEMPLATES.md @@ -0,0 +1,84 @@ +<!-- m-wiki: type=top-level slug=dockerfile-templates topic=null base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: top-level. 0 sources. + +# Dockerfile Templates + +Dockerfile templates live in `resources/com/meesho/`. Each build stage calls `constructTemplate.renderTemplate()` to select and render the appropriate template with service-specific values. + +## TL;DR + +- Templates use Groovy's `SimpleTemplateEngine` — `${}` substitutions from a `Map binding`. +- `constructTemplate.renderTemplate(binding, templateFile, outputPath)` calls `libraryResource` to load the template, renders it, and writes the output file. +- Template selection is by `dockerBuildVersion` string: `maven-21`, `go-1.22`, `node-20`, etc. +- Output is always written to the workspace and then used by the Docker build step. +- `_construct()` is annotated `@NonCPS` because `SimpleTemplateEngine` is not CPS-serializable. + +## Mental model + +`constructTemplate` is a thin wrapper around Groovy's standard template engine. The binding map contains all deployment metadata (repo name, version tag, environment, JVM flags, etc.) assembled by the build stage and `enable_backward_compatibility()`. The template file itself is stored as a Jenkins library resource and loaded via `libraryResource`. + +## Structure / data flow + +``` +Dockerfile template selection (in each build stage): + dockerBuildVersion = "go-1.22" + │ + ▼ +constructTemplate.renderTemplate( + binding = { repo_name, tag, env, ... }, + templateFile = "go-1.22-Dockerfile", (or "go-Dockerfile" depending on version) + outputPath = "${WORKSPACE}/Dockerfile" +) + │ + ▼ +libraryResource "com/meesho/go-1.22-Dockerfile" + │ + ▼ +SimpleTemplateEngine.createTemplate(text).make(binding) + │ + ▼ +writeFile(Dockerfile) → docker build -f Dockerfile -t <registry>/<repo>:<tag> . +``` + +Available templates (from `resources/com/meesho/`): +``` +maven-Dockerfile maven-21-Dockerfile +go-Dockerfile go-1.22-Dockerfile +node-Dockerfile node-18-Dockerfile node-20-Dockerfile +python-Dockerfile python-3.7-Dockerfile python-3.10-Dockerfile python-3.12-Dockerfile +rust-Dockerfile +php-Dockerfile +argoApp.yaml (ArgoCD Application manifest template) +values.yaml node-values.yaml go-values.yaml python-values.yaml +cron-values.yaml php-values.yaml +``` + +## Key code locations + +| Symbol | File | What it does | +|--------|------|--------------| +| `renderTemplate` | `src/com/meesho/utilities/constructTemplate.groovy:renderTemplate` | Loads library resource + renders + writes file | +| `_construct` | `src/com/meesho/utilities/constructTemplate.groovy:_construct` | `@NonCPS` template rendering via SimpleTemplateEngine | +| `get_value_yaml_file` | `src/com/meesho/stages/deployArgoCD.groovy:get_value_yaml_file` | Selects which values.yaml template to use for Helm | +| `get_default_command` | `src/com/meesho/stages/deployArgoCD.groovy:get_default_command` | Default container command per build version | + +## Sharp edges + +- **`_construct` is `@NonCPS`** (`src/com/meesho/utilities/constructTemplate.groovy:_construct`): this means it cannot access Jenkins pipeline steps (e.g., `echo`, `sh`) or `env.*` inside the method. All values must be in the `binding` map. Any caller that invokes `_construct` inside a `parallel` block or closure must ensure the closure itself is also `@NonCPS` or does not cross a serialisation boundary. Do not move `_construct` into a CPS context — keep the annotation and call it from a CPS-safe wrapper. +- **`binding` is copied defensively**: `_construct` wraps the incoming map in `new HashMap(binding)` before passing to the engine, preventing mutation of the caller's map. +- **`argoApp.yaml` is also a template**: the ArgoCD Application manifest is rendered the same way as Dockerfiles. This means ArgoCD app metadata (cluster, namespace, Helm chart path) is all driven by the deployment YAML binding. +- **Values templates select by `dockerBuildVersion`**: `get_value_yaml_file` maps `node-*` → `node-values.yaml`, `python-*` → `python-values.yaml`, etc. An unrecognized version returns `null` and the pipeline fails. + +## Related concepts + +- [Language builds](07-LANGUAGE-BUILDS.md) — which stage calls renderTemplate +- [Deploy ArgoCD](04-DEPLOY-ARGOCD.md) — uses renderTemplate for both ArgoCD app and Helm values + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Previous](07-LANGUAGE-BUILDS.md) · [Index](../index.md) · [Next →](09-INFRA-PODS.md) diff --git a/docs/wiki/pages/09-INFRA-PODS.md b/docs/wiki/pages/09-INFRA-PODS.md new file mode 100644 index 0000000..62b2972 --- /dev/null +++ b/docs/wiki/pages/09-INFRA-PODS.md @@ -0,0 +1,81 @@ +<!-- m-wiki: type=top-level slug=infra-pods topic=null base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: top-level. 0 sources. + +# Infra: GCP Jenkins Agent Pods + +All GCP builds run inside a Jenkins Kubernetes pod provisioned from YAML specs in `resources/org/meesho/`. The pod YAML is selected by `env.INFRA_ENV` and loaded via `libraryResource` in `eksCICD.gcpInfra()`. + +## TL;DR + +- Pod specs live in `resources/org/meesho/*.yaml`. +- Filename pattern: `<INFRA_ENV>-pod.yaml` and `<INFRA_ENV>-sidecar-pod.yaml`. +- `INFRA_ENV` values seen in practice: `prd`, `stg`, `dev`, `toolchain`. +- All builds run in the `devops-tools` container defined in the pod spec. +- Sidecar pods (`useSidecar: true` in consumer Jenkinsfile) add extra containers alongside `devops-tools`. +- AWS builds skip this entirely — they run directly on static EKS nodes labeled `EKS`. + +## Mental model + +Jenkins provisions a fresh Kubernetes pod for each build. The pod spec defines: +- Which container images to run (typically `devops-tools` + optional sidecar). +- Resource requests/limits for the build container. +- Any mounted volumes (e.g., Docker socket for DinD builds). + +`gcpInfra()` reads `env.INFRA_ENV` (injected by the Jenkins job configuration), constructs the pod YAML filename, loads it via `libraryResource`, and wraps the entire pipeline in a `podTemplate { node { container('devops-tools') { ... } } }` block. + +## Structure / data flow + +``` +eksCICD.call(repo): + CLOUD_PROVIDER=GCP → gcpInfra(repo) + +gcpInfra(repo): + isSidecarNeeded = repo.get('useSidecar', false) + yamlName = isSidecarNeeded ? "${INFRA_ENV}-sidecar-pod.yaml" : "${INFRA_ENV}-pod.yaml" + podyaml = "org/meesho/${yamlName}" + podTemplate(yaml: libraryResource(podyaml)) { + node(POD_LABEL) { + container('devops-tools') { + commonCICDFlow(repo) + } + } + } + +Pod YAML inventory (resources/org/meesho/): + prd-pod.yaml prd-sidecar-pod.yaml + stg-pod.yaml stg-sidecar-pod.yaml + dev-pod.yaml dev-sidecar-pod.yaml + toolchain-pod.yaml toolchain-sidecar-pod.yaml +``` + +## Key code locations + +| Symbol | File | What it does | +|--------|------|--------------| +| `gcpInfra` | `vars/eksCICD.groovy:gcpInfra` | Selects pod YAML and wraps pipeline in podTemplate | +| `awsInfra` | `vars/eksCICD.groovy:awsInfra` | AWS path — no pod YAML, just node('EKS') | +| `run` | `src/com/meesho/utilities/nodePoolSelection.groovy:run` | Selects GKE node pool label for deployment (separate from build pod) | + +## Sharp edges + +- **Build pod vs deployment node pool are different things**: the Jenkins agent pod is where the build runs; `nodePoolSelection.run()` selects which GKE node pool the deployed *application* should land on. These are independent. +- **`INFRA_ENV` is job-level config**: it is not derived from the branch name. It is injected by the Jenkins multibranch pipeline configuration. A mis-configured job can run a `main` branch build with `INFRA_ENV=dev` — which would use the dev pod spec. +- **Toolchain pods are segregated**: `toolchain-pod.yaml` uses images from the `toolchain` namespace in `meesho-central-dev-0622` project — separate from standard build pods. +- **DinD (Docker-in-Docker) socket**: the `devops-tools` container in pod specs mounts a DinD service socket (`env.DOCKER_HOST`) rather than the host Docker socket. The `DOCKER_HOST` env var is set by `constructParam.run()` based on `cicd_environment`. The correct endpoint values are `dind-prd-svc` (prd) and `dind-dev-new-svc.jenkins-new.svc.cluster.local` (stg/ftr). Do not use `tcp://localhost:2375` — it creates conflicts that require unnecessary guard logic. The DinD image must come from the internal GAR registry (`asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/docker:28-dind`) — not Docker Hub. +- **`build-tools` image version encodes the Go compiler and sonar-scanner CLI**: `resources/org/meesho/prd-pod.yaml` and `stg-pod.yaml` reference a `build-tools` image that bundles Go, `sonar-scanner-cli`, and other toolchain binaries. When a new toolchain binary is needed, bump the image tag in both pod YAMLs — never `curl`/`wget` a tool from the internet inside a pipeline stage. +- **Non-prd node pools are BU-scoped**: `stg`/`ftr`/`dev` pods land on `{BU}-shared` pools, so all services in the same BU share one pool. A memory leak or noisy-neighbour in one `supply` service degrades all other `supply` services on staging. + +## Related concepts + +- [Node pool selection](infra/node-pool-selection.md) — GKE node pool assignment for deployed applications +- [Architecture](01-ARCHITECTURE.md) — where gcpInfra fits in the pipeline +- [Config policy](06-CONFIG-POLICY.md) — env.DOCKER_HOST set by constructParam + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Previous](08-DOCKERFILE-TEMPLATES.md) · [Index](../index.md) · [Next →](10-NOTIFICATIONS.md) diff --git a/docs/wiki/pages/10-NOTIFICATIONS.md b/docs/wiki/pages/10-NOTIFICATIONS.md new file mode 100644 index 0000000..d4cffa7 --- /dev/null +++ b/docs/wiki/pages/10-NOTIFICATIONS.md @@ -0,0 +1,82 @@ +<!-- m-wiki: type=top-level slug=notifications topic=null base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: top-level. 0 sources. + +# Notifications + +`notify.groovy` runs in the pipeline's `finally` block — it always executes regardless of build outcome. It routes to different backends based on who triggered the build and what environment the build ran in. + +## TL;DR + +- `ringmaster-bot` or `turbo-turtle` triggered → calls `deployRingmaster.run()` + Slack on `prd`. +- All other users → direct Slack `slackSend` only. +- Toolchain env → HTTP callback to `172.23.72.116:5002/api/v1/deploy/build-callback` (skip Slack/Ringmaster). +- On `prd` push builds, also posts to `deployment-tracker` API and Ringmaster deployment history. +- `notify_channel` comes from `config.yaml`; defaults to `ci-cd-status` if empty. + +## Mental model + +`notify.run()` is the final stage in `commonCICDFlow`. Since it's in a `finally` block, `env.msg` carries the final build status message (set by each catch block in `commonCICDFlow`). The notification backend selection depends on the triggering user, not on the environment: + +- `ringmaster-bot` → Ringmaster API (`/api/v1/key/cicd/cd/update`) with a token-authenticated payload. +- Turbo-Turtle or other allowed users → Turbo-Turtle callback (`/api/v1/ci/jenkins/callback`) via a temporary JSON file to avoid shell quoting issues. + +## Structure / data flow + +``` +notify.run(config): + ├─ [toolchain] if env.INFRA_ENV == 'toolchain': + │ POST http://172.23.72.116:5002/api/v1/deploy/build-callback + │ return (skip everything below) + │ + ├─ build_user = getCause(UserIdCause).getUserId() + │ + ├─ if build_user in [ringmaster-bot, turbo-turtle]: + │ deployRingmaster.run(...) → POST to Ringmaster or Turbo-Turtle + │ if prd: slackSend(notify_channel, deploy URL) + │ skip_notify = true + │ + ├─ if !skip_notify: + │ slackSend(notify_channel, job name + build number + tag + msg) + │ + └─ if main/master branch: + postTrackingApi(config) ← deployment-tracker API + postTrackingRingmasterApi(config) ← Ringmaster deployment history + +deployRingmaster.run(repo_name, deployment_order, tag, ...): + if build_user == "ringmaster-bot": + POST https://ringmaster-api.meeshogcp.in/api/v1/key/cicd/cd/update + else: + POST http://turbo-turtle.meeshogcp.in/api/v1/ci/jenkins/callback +``` + +## Key code locations + +| Symbol | File | What it does | +|--------|------|--------------| +| `run` | `src/com/meesho/stages/notify.groovy:run` | Main notify dispatcher | +| `postTrackingApi` | `src/com/meesho/stages/notify.groovy:postTrackingApi` | Posts to deployment-tracker API | +| `postTrackingRingmasterApi` | `src/com/meesho/stages/notify.groovy:postTrackingRingmasterApi` | Posts to Ringmaster deployment history | +| `run` | `src/com/meesho/stages/deployRingmaster.groovy:run` | Routes to Ringmaster or Turbo-Turtle callback | +| `callApi` | `src/com/meesho/stages/deployRingmaster.groovy:callApi` | Ringmaster API call with `ringmaster-token` credential | + +## Sharp edges + +- **JSON payload via temp file** (Turbo-Turtle path): the Turbo-Turtle curl call writes the payload to a temp file (`cicd_payload_${BUILD_NUMBER}_${ts}.json`) to avoid shell quoting issues with JSON special characters. The file is always deleted in a `finally` block. +- **`prd` vs `int` API endpoints**: for both Ringmaster and deployment-tracker, `prd` and `int` share the `ringmaster-api.meeshogcp.in` endpoint while `stg`/`ftr` use `ringmaster-api.admin.meeshogcp.in`. +- **`skip_notify` is `env.*` not `config.*`**: it's set by `constructParam.run()` (always `true` for GCP) and can also be set by the consumer `config.yaml`. Both must be false for Slack to fire. +- **`env.msg` is the error channel**: each `catch` block in `commonCICDFlow` sets `env.msg` before the `finally` block calls `notify`. If nothing failed, `env.msg` stays `'Job Passed'`. + +## Related concepts + +- [Ringmaster integration](deploy/ringmaster-integration.md) — detailed routing logic +- [Architecture](01-ARCHITECTURE.md) — where notify fits in commonCICDFlow +- [Environment mapping](05-ENVIRONMENT-MAPPING.md) — cicd_environment determines API base URL + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Previous](09-INFRA-PODS.md) · [Index](../index.md) · [Next →](10-NOTIFICATIONS.md) diff --git a/docs/wiki/pages/adr/adr-index.md b/docs/wiki/pages/adr/adr-index.md new file mode 100644 index 0000000..e2cd3c3 --- /dev/null +++ b/docs/wiki/pages/adr/adr-index.md @@ -0,0 +1,55 @@ +<!-- m-wiki: type=concept slug=adr-index topic=adr base-sha=d6708eca4236 generated-at=2026-05-12T12:00:00+00:00 sources=[docs/adr/README.md, docs/adr/0001-single-shared-library-for-all-services.md, docs/adr/0002-branch-name-as-sole-environment-selector.md, docs/adr/0003-policy-exceptions-in-separate-whitelist-repo.md, docs/adr/0004-fresh-whitelist-clone-per-build.md, docs/adr/0005-config-only-change-detection-skip-build.md, docs/adr/0006-ringmaster-mandatory-build-trigger-gate.md, docs/adr/0007-gitops-via-argocd-4-step-sync-sequence.md, docs/adr/0008-canary-mandatory-for-tier1-services-in-prd.md, docs/adr/0009-jvm-heap-auto-derived-from-pod-memory-request.md] --> + +> Generated 2026-05-12 at base-sha d6708eca4236. Type: concept. 10 sources. + +# Architecture Decision Records — Index + +devops-lib's ADRs capture the "why" behind the library's design — the decisions that would otherwise be tribal knowledge. All 9 ADRs were retroactively documented on 2026-05-12 from codebase analysis and developer interviews. All are **Status: Accepted** and assessed as still appropriate. + +## Why ADRs matter here + +devops-lib has several decisions whose rationale is non-obvious: why does the whitelist get re-cloned on every build (not cached)? Why is the 4-step ArgoCD sequence non-interchangeable? Why must all builds go through Ringmaster? Without ADRs, engineers modifying the library break load-bearing constraints without realising it. + +## Decision inventory + +| # | Decision | Category | Key insight | +|---|----------|----------|-------------| +| [ADR-0001](../../../adr/0001-single-shared-library-for-all-services.md) | Single shared library for all services | PATTERN | Policy enforcement must be uniform; per-team Jenkinsfiles produce drift | +| [ADR-0002](../../../adr/0002-branch-name-as-sole-environment-selector.md) | Branch name as sole environment selector | PATTERN | Branch IS the environment contract; per-service env config creates misconfiguration risk | +| [ADR-0003](../../../adr/0003-policy-exceptions-in-separate-whitelist-repo.md) | Policy exceptions in Meesho/whitelists repo | PATTERN | Service teams cannot self-grant bypasses; all exceptions require DevOps review | +| [ADR-0004](../../../adr/0004-fresh-whitelist-clone-per-build.md) | Fresh whitelist clone per build | PATTERN | Policy changes must take effect on the very next build without a library release | +| [ADR-0005](../../../adr/0005-config-only-change-detection-skip-build.md) | Config-only change detection — skip build | PATTERN | No source change → no new binary; reuse the latest image tag from GAR | +| [ADR-0006](../../../adr/0006-ringmaster-mandatory-build-trigger-gate.md) | Ringmaster mandatory trigger gate | PATTERN | Every deployment must be tracked; direct Jenkins triggers bypass Ringmaster's ledger | +| [ADR-0007](../../../adr/0007-gitops-via-argocd-4-step-sync-sequence.md) | GitOps via 4-step ArgoCD sync sequence | INFRA | Steps 2 and 4 are non-interchangeable; swapping them causes silent sync failures | +| [ADR-0008](../../../adr/0008-canary-mandatory-for-tier1-services-in-prd.md) | Canary mandatory for Tier-1 (sp0/up0) in prd | RELIABILITY | Incident-driven: non-canary prd deploys for critical services caused outages | +| [ADR-0009](../../../adr/0009-jvm-heap-auto-derived-from-pod-memory-request.md) | JVM heap auto-derived from pod memory_request | RELIABILITY | OOM incident remediation: auto-calc prevents under-sizing; `jvm_memory_override: true` escapes it | + +## Decisions that are load-bearing constraints + +These three decisions have "never change without understanding this" consequences: + +**ADR-0004 (fresh whitelist clone)** — `getWhitelistedRepos` in `src/com/meesho/utilities/constructParam.groovy:getWhitelistedRepos` re-clones `Meesho/whitelists` on every call by design. Never add a cache — it would prevent immediate policy enforcement. + +**ADR-0006 (Ringmaster gate)** — The string `"ringmaster-bot"` at `vars/eksCICD.groovy:call` is the sole routing signal between Ringmaster and Turbo-Turtle callbacks. Never rename it without coordinating with both teams. + +**ADR-0007 (4-step ArgoCD sequence)** — Steps 2 (`refresh_app_of_apps`) and 4 (`refresh_and_sync`) in `src/com/meesho/stages/deployArgoCD.groovy:run` are non-interchangeable. For a first-deploy, step 2 must create the Application object before step 4 can sync it. + +## When to read which ADR + +- **Debugging a build not triggering**: ADR-0006 (Ringmaster gate) +- **Debugging a deploy that failed on ArgoCD sync**: ADR-0007 (4-step sequence) +- **Understanding why policy changes take effect immediately**: ADR-0004 (fresh clone) +- **Service team asking for a sonar bypass**: ADR-0003 (whitelist repo) +- **Canary enforcement failing for a prd deploy**: ADR-0008 +- **OOM in a Java service pod**: ADR-0009 + +## Related concepts + +- [Whitelist system](../policy/whitelist-system.md) — runtime whitelist enforcement (ADR-0003, ADR-0004) +- [ArgoCD sync](../deploy/argocd-sync.md) — step-by-step walkthrough (ADR-0007) +- [Ringmaster integration](../deploy/ringmaster-integration.md) — callback flow (ADR-0006) +- [Environment mapping](../05-ENVIRONMENT-MAPPING.md) — branch → environment (ADR-0002) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> diff --git a/docs/wiki/pages/build/build-dispatch.md b/docs/wiki/pages/build/build-dispatch.md new file mode 100644 index 0000000..2173ef1 --- /dev/null +++ b/docs/wiki/pages/build/build-dispatch.md @@ -0,0 +1,53 @@ +<!-- m-wiki: type=concept slug=build-dispatch topic=build base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# Build Dispatch + +`buildObjHelper.run(build_tool)` maps a `build_tool` string from `config.yaml` to a concrete builder class instance using a Groovy `switch/case` with regex patterns. + +## Where it applies in this repo + +`src/com/meesho/stages/buildObjHelper.groovy:run` + +The full dispatch table (in order, first match wins): + +| Pattern | Builder class | +|---------|--------------| +| `maven` (exact) | `buildMaven` | +| `docker` (exact) | `buildDocker` | +| `~/^maven-.*/` | `buildMaven` | +| `~/^python-.*/` | `buildPython` | +| `~/^node-.*/` | `buildNode` | +| `~/^rust.*/` | `buildRust` | +| `~/^go.*/` | `buildGo` | +| `gradle` (exact) | `buildGradle` | +| `php` (exact) | `buildPhp` | +| default | `defaultBuild()` | + +Each builder is instantiated fresh per build — no shared state between builds. + +## Why this design + +Groovy `switch/case` evaluates patterns top-to-bottom and returns on the first match. Regex patterns (the `~/…/` syntax) cover version-suffixed variants like `go-1.22`, `node-20`, `python-3.12` without requiring an exhaustive case list. The exact-match cases for `maven` and `docker` appear before the regex catch-all `~/^maven-.*/` to handle the legacy bare-string case. + +If no case matches, `defaultBuild()` is called without logging a warning — the pipeline reports success with no artifact produced. A typo in `config.yaml` (e.g. `golang` instead of `go`) produces this silent no-op. If a build succeeds but produces no Docker image, check `build_tool` spelling in `config.yaml` first. + +## Related + +- [Build stages](../03-BUILD-STAGES.md) — broader build lifecycle +- [Language builds](../07-LANGUAGE-BUILDS.md) — per-language builder details + +## Sources + +(no raw/ sources at bootstrap) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/build/config-only-detection.md b/docs/wiki/pages/build/config-only-detection.md new file mode 100644 index 0000000..806ad12 --- /dev/null +++ b/docs/wiki/pages/build/config-only-detection.md @@ -0,0 +1,46 @@ +<!-- m-wiki: type=concept slug=config-only-detection topic=build base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# Config-Only Change Detection + +When a commit only changes YAML config files (no source code), each language builder skips the compile/test/image-build steps and instead fetches the last-built image tag from GAR (Google Artifact Registry). This avoids rebuilding identical binaries when only configs changed. + +## Where it applies in this repo + +Implemented independently in each language builder (`src/com/meesho/stages/buildMaven.groovy`, `buildGo.groovy`, `buildNode.groovy`, etc.). + +The detection pattern varies slightly by builder but follows the same logic: + +1. Run `git diff HEAD~1 -- configs/` (or `git diff ORIG_HEAD -- configs/`). +2. If the diff is non-empty AND no source files changed → set `appConfigChanges = true`, skip compilation. +3. Fetch the latest image tag from the artifact bucket (S3 or GCS) or GAR. +4. Set `env.TAG` to the fetched tag. +5. Proceed directly to `deployArgoCD.run()`. + +The check is gated by `skip_s3_check` in `config.yaml`. Setting `skip_s3_check: true` forces a full rebuild even when only configs changed. + +## Why this design + +Config-only deployments are common at Meesho (dynamic config updates, feature flags). Rebuilding the entire Java or Go binary for a one-line YAML change wastes 3–10 minutes. By reusing the last image tag and skipping straight to ArgoCD deploy, the pipeline completes in ~1 minute for config-only changes. + +The detection relies on `git diff` against the previous commit, so it only works when the commit history is linear. Force-pushes or squash merges may produce false negatives (full rebuild triggered unnecessarily). + +## Related + +- [Language builds](../07-LANGUAGE-BUILDS.md) — where this check is embedded per builder +- [Build dispatch](build-dispatch.md) — the builder instance that contains this check + +## Sources + +(no raw/ sources at bootstrap) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/build/docker-tagging.md b/docs/wiki/pages/build/docker-tagging.md new file mode 100644 index 0000000..e3c37b0 --- /dev/null +++ b/docs/wiki/pages/build/docker-tagging.md @@ -0,0 +1,57 @@ +<!-- m-wiki: type=concept slug=docker-tagging topic=build base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# Docker Tagging + +`getDockerParams.getTag()` produces the Docker image tag for each build. The tag format encodes the version, commit SHA, and a timestamp to guarantee uniqueness across rebuilds of the same commit. + +## Where it applies in this repo + +`src/com/meesho/utilities/getDockerParams.groovy:getTag` + +**Standard builds:** +``` +v{pom-version or package-version}-{7-char-sha}-{epoch-ms} +``` +Example: `v1.4.2-a3f8c21-1714920000000` + +**Toolchain builds** (when `env.INFRA_ENV == 'toolchain'`): +``` +v{version}-{7-char-sha} +``` +Example: `v1.4.2-a3f8c21` + +The short SHA comes from `git log -1 --format=%h` (7 chars). The full 40-char SHA is also captured in `env.commit_id` as a side effect of `getCommitid()`. + +Version is read from: +- `pom.xml` → `xq -r .project.version pom.xml` +- `package.json` → `jq -r .version package.json` +- Default: `1.0` + +**Hotfix Maven builds** override the tag format to `v{version}-HOT` (set directly in `buildMaven.run()`, not via `getTag`). + +## Why this design + +The epoch-ms suffix ensures that two builds from the exact same commit produce different tags. This is intentional: if a build fails mid-way and is retried, the retry must produce a new image (the previous one may be partially pushed or broken). Without the timestamp, `docker push` on a retry would be a no-op if the tag already exists. + +Toolchain builds omit the timestamp because toolchain images are content-addressed: the same source commit must always produce the same tag so toolchain consumers can pin to a stable reference without tracking timestamps. + +## Related + +- [Language builds](../07-LANGUAGE-BUILDS.md) — each builder sets `env.TAG` using getTag +- [Notifications](../10-NOTIFICATIONS.md) — `env.TAG` is included in Ringmaster and Slack payloads + +## Sources + +(no raw/ sources at bootstrap) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/build/node-paired-files.md b/docs/wiki/pages/build/node-paired-files.md new file mode 100644 index 0000000..c058dc5 --- /dev/null +++ b/docs/wiki/pages/build/node-paired-files.md @@ -0,0 +1,49 @@ +<!-- m-wiki: type=concept slug=node-paired-files topic=build base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# Node Build: Paired File Rule + +`src/com/meesho/stages/buildNode.groovy` and `resources/com/meesho/node-Dockerfile` are paired files. Changes to Node install logic in one must be mirrored in the other to avoid a split-brain build path. + +## Where it applies in this repo + +`src/com/meesho/stages/buildNode.groovy:buildDckr` +`resources/com/meesho/node-Dockerfile` + +## How they interact + +`buildNode.groovy` detects the package manager at runtime (npm vs pnpm vs yarn) and passes the install command to the Dockerfile as the `npm_install_arg` template variable. The Dockerfile's `else` branch handles the fallback when no explicit or detected command is provided (currently `npm ci`). + +``` +buildNode.buildDckr(config): + ├─ Detect package manager → npm_install_arg = "pnpm install" | "npm ci" | ... + ├─ constructTemplate.renderTemplate(binding, 'node-Dockerfile') + │ └─ binding.npm_install_arg → substituted into node-Dockerfile + └─ docker build -f Dockerfile ... +``` + +When `npm_install_arg` is not set in `config.yaml`, `buildNode.groovy` falls back to its own detection logic. The Dockerfile default branch handles the case where detection produces nothing. + +## Why this matters + +If `buildNode.groovy` changes the fallback install command or adds support for a new package manager, the Dockerfile default branch must be updated in the same PR. Changing only one file leaves them out of sync: the runtime path may succeed while the Docker fallback uses the old command (or vice versa). This creates subtly different images depending on whether `npm_install_arg` is explicitly configured. + +## Related + +- [Language builds](../07-LANGUAGE-BUILDS.md) — Node builder overview +- [Dockerfile templates](../08-DOCKERFILE-TEMPLATES.md) — how renderTemplate works + +## Sources + +(no raw/ sources) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/build/scm-variable-scope.md b/docs/wiki/pages/build/scm-variable-scope.md new file mode 100644 index 0000000..e38d762 --- /dev/null +++ b/docs/wiki/pages/build/scm-variable-scope.md @@ -0,0 +1,54 @@ +<!-- m-wiki: type=concept slug=scm-variable-scope topic=build base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# SCM Variable Scope + +The `scm` variable is only available inside consumer Jenkinsfiles, not inside `vars/` or `src/` of the shared library. Referencing it in library code causes a runtime `MissingPropertyException`. + +## Where it applies in this repo + +`vars/eksCICD.groovy` and all files under `src/com/meesho/` + +## Why `scm` is unavailable + +The `scm` variable (branch, remote URL, credentials) is injected by the GitHub Branch Source plugin into **consumer Jenkinsfiles** at the time they are loaded by Jenkins. Library code (everything under `vars/` and `src/`) is a separate classloader context — the plugin does not inject `scm` there. + +Attempting to access `scm.branches` or `scm.userRemoteConfigs` from library code will throw: + +``` +MissingPropertyException: No such property: scm for class: groovy.lang.Binding +``` + +## Correct alternatives + +Inside library code, use the Jenkins-injected environment variables instead: + +| Need | Use instead of `scm.*` | +|------|------------------------| +| Branch name | `env.BRANCH_NAME` | +| Repository URL | `env.GIT_URL` | +| PR number | `env.CHANGE_ID` | +| PR source branch | `env.CHANGE_BRANCH` | +| PR target branch | `env.CHANGE_TARGET` | + +All of these are set by the GitHub Branch Source plugin before library code runs. + +## Related + +- [Entry points](../02-ENTRYPOINTS.md) — where this constraint applies +- [Environment mapping](../05-ENVIRONMENT-MAPPING.md) — `env.CHANGE_ID` is the PR-build detector + +## Sources + +(no raw/ sources) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/concepts/observability.md b/docs/wiki/pages/concepts/observability.md new file mode 100644 index 0000000..e6a44e8 --- /dev/null +++ b/docs/wiki/pages/concepts/observability.md @@ -0,0 +1,55 @@ +<!-- m-wiki: type=concept slug=observability topic=concepts base-sha=5399a5ddc36b generated-at=2026-05-21 sources=[code:vars/log.groovy] --> + +> Generated 2026-05-21 at base-sha 5399a5ddc36b. Type: concept. 1 source. + +# Logging via `log.groovy` + +[`vars/log.groovy`](../../../../vars/log.groovy) is the closest thing this codebase has to a logger. It's a 24-line Groovy file that wraps `echo` with ANSI colour codes. + +## The three calls + +```groovy +log.info(msg) // → echo "${GREEN}INFO: ${msg}${BLACK}" +log.warning(msg) // → echo "${RED}WARNING: ${msg}${BLACK}" +log.error(msg) // → echo "${RED}ERROR: ${msg}${BLACK}" +``` + +That's the whole API. There is no severity level configuration, no structured fields (no JSON), no destination other than the Jenkins console. + +## What `log.info` gives you over bare `echo` + +- **ANSI colour** — pipeline output in the Jenkins UI is easier to scan when INFO is green and ERROR is red. +- **Consistent prefix** (`INFO: ` / `WARNING: ` / `ERROR: `) — a `grep ERROR pipeline.log` pattern works across every service. +- **Nothing else.** No timestamps (Jenkins adds those via `timestamps` wrapper), no caller tracing, no correlation ID. + +## When `bare echo` is acceptable + +Lots of existing code uses bare `echo`. The mix is mostly historical. New code should prefer `log.info` for normal messages and `log.error` for failures, but you'll see bare `echo` in: + +- One-off banners (`echo "=========="`). +- Single-line status (`echo "Building module: ${m}"` inside a loop). +- ANSI-coloured ad-hoc messages where the code path needs a one-off colour (some `vars/eksCICD.groovy` and `vars/buildPipeline.groovy` lines do this). + +There is no policy that bans bare `echo`. The reconcile run's `BUGS_AND_IMPROVEMENTS_REPORT.md` does list "inconsistent logging" as a P1 item but no rule has been graduated into `CLAUDE.md` NEVER DO yet. + +## `env.msg` as failure state + +A separate convention: most stages set `env.msg` to a human-readable failure reason before re-throwing, e.g. [`buildNode.groovy:21`](../../../../src/com/meesho/stages/buildNode.groovy): + +```groovy +env.msg = 'Error in building node packages...' +log.error(env.msg) +currentBuild.result = env.FAILURE +throw e +``` + +`notify.groovy` then reads `env.msg` (and the related `env.error_msg_to_db`) when composing the Slack message and the deployment-tracker payload. **Don't rename `env.msg`** — too many callers read it. [`deployRingmaster.groovy`](../../../../src/com/meesho/stages/deployRingmaster.groovy) catch blocks are inconsistent here: some set `env.msg`, others don't — flagged in `BUGS_AND_IMPROVEMENTS_REPORT.md` as standardisation work. + +## What this codebase doesn't have + +- No structured logging (no JSON, no key-value pairs, no trace IDs). +- No log level filtering — every `log.info` always prints. +- No log forwarding to anything other than the Jenkins console. +- No "audit log" stream separate from the pipeline log. + +If you need to instrument a pipeline run for external observability, the current convention is to POST to a downstream system directly (see how `notify.groovy` calls the Deployment Tracker at lines 108-152). There is no shared metric/event emitter. diff --git a/docs/wiki/pages/concepts/secrets-and-auth.md b/docs/wiki/pages/concepts/secrets-and-auth.md new file mode 100644 index 0000000..3b5ef0a --- /dev/null +++ b/docs/wiki/pages/concepts/secrets-and-auth.md @@ -0,0 +1,51 @@ +<!-- m-wiki: type=concept slug=secrets-and-auth topic=concepts base-sha=5399a5ddc36b generated-at=2026-05-21 sources=[code:src/com/meesho/utilities/addSSHKey.groovy, code:src/com/meesho/stages/buildNode.groovy, code:src/com/meesho/stages/buildGo.groovy, code:src/com/meesho/stages/deployArgoCD.groovy, code:src/com/meesho/stages/securityScan.groovy, code:src/com/meesho/stages/notify.groovy] --> + +> Generated 2026-05-21 at base-sha 5399a5ddc36b. Type: concept. 6 sources. + +# Secrets, SSH keys, and downstream auth + +How the pipeline authenticates to every external system, in one place. Every entry below comes from a real `withCredentials { ... }` block or env var read in the current code. + +## Downstreams + auth mechanisms + +| Downstream | Credential / mechanism | Code site | +|---|---|---| +| GitHub clone/push (HTTPS) | `env.GITHUB_CRED = 'svc-devops-meesho'` (`gitUsernamePassword`) | [`gitActions.groovy:13`](../../../../src/com/meesho/utilities/gitActions.groovy), [`buildGo.groovy:300`](../../../../src/com/meesho/stages/buildGo.groovy), [`buildNode.groovy:413`](../../../../src/com/meesho/stages/buildNode.groovy) | +| GitHub clone (SSH, private repos) | `credentialsId: 'ssh-private-key'` written to `./id_github_jenkins` (0600) | [`addSSHKey.groovy:4-5`](../../../../src/com/meesho/utilities/addSSHKey.groovy) | +| Vault (GCP secrets) | `env.vaultToken` string | [`buildNode.groovy:555`](../../../../src/com/meesho/stages/buildNode.groovy), [`constructParam.groovy:170,175`](../../../../src/com/meesho/utilities/constructParam.groovy) (`vault-prd.meeshogcp.in`, `vault-dev.meeshogcp.in`) | +| JFrog (Maven deploy) | `-DuseProdRepo=true` / `-DuseTestRepo=true` Maven profile | [`onlyPushtoJfrog.groovy:43-47`](../../../../vars/onlyPushtoJfrog.groovy) | +| GCP Docker registry | `gcloud auth configure-docker` (SDK ambient auth) | [`buildGo.groovy:126`](../../../../src/com/meesho/stages/buildGo.groovy), [`buildNode.groovy:374`](../../../../src/com/meesho/stages/buildNode.groovy) | +| AWS ECR | `aws ecr get-login-password ... \| docker login --password-stdin` | [`buildNode.groovy:371`](../../../../src/com/meesho/stages/buildNode.groovy), [`buildPython.groovy:93`](../../../../src/com/meesho/stages/buildPython.groovy), [`buildMaven.groovy:566`](../../../../src/com/meesho/stages/buildMaven.groovy), [`buildGradle.groovy:520`](../../../../src/com/meesho/stages/buildGradle.groovy), [`buildPhp.groovy:59`](../../../../src/com/meesho/stages/buildPhp.groovy) | +| ArgoCD | `env.argoCreds` (`usernamePassword`); `argocd login ${env.argoURL}:443` | [`deployArgoCD.groovy:490, 522`](../../../../src/com/meesho/stages/deployArgoCD.groovy) | +| npm registry | `.npmrc` from AWS Secrets Manager or Vault → written to workspace | [`buildNode.groovy:63-68, 337`](../../../../src/com/meesho/stages/buildNode.groovy) | +| SonarQube | `env.sonarToken` string; `withSonarQubeEnv { ... }` against `sonarqube-prd` | [`buildMaven.groovy:245-251`](../../../../src/com/meesho/stages/buildMaven.groovy), [`buildNode.groovy:406`](../../../../src/com/meesho/stages/buildNode.groovy) | +| Ringmaster | `credentialsId: 'ringmaster-token'` (`usernamePassword`) | [`deployRingmaster.groovy:115`](../../../../src/com/meesho/stages/deployRingmaster.groovy), [`notify.groovy:117`](../../../../src/com/meesho/stages/notify.groovy) | + +## The SSH key write path + +[`addSSHKey.groovy:3-7`](../../../../src/com/meesho/utilities/addSSHKey.groovy) writes the credential file inside `withCredentials { ... }`: + +```groovy +withCredentials([sshUserPrivateKey(credentialsId: 'ssh-private-key', keyFileVariable: 'FILE')]) { + sh "cat ${FILE} > ./id_github_jenkins; chmod 600 ./id_github_jenkins; ..." +} +``` + +There is a **race window** between the `cat` write and the `chmod` — a co-resident process could read the file with default umask permissions for that brief interval. There is also **no cleanup** of `./id_github_jenkins` after use. Both are flagged in `BUGS_AND_IMPROVEMENTS_REPORT.md`. + +The key file is **NOT cat'd to stdout / logs** — earlier PR-review concerns (PR #634) about that pattern have been remediated; the current `cat ${FILE} > ./id_github_jenkins` is a file write, not a print. See [`review-learnings.md`](../../../../review-learnings.md) for the historical trail. + +## NEVER DO + +- **Never print or `cat` an SSH private key to stdout / logs.** Always go through `withCredentials` + a 0600 file. (Graduated rule — see [`CLAUDE.md`](../../../../CLAUDE.md) NEVER DO.) +- **Never hard-code bare IPs as curl/HTTP targets.** Use DNS hostnames. Known existing violation: [`securityScan.groovy:11`](../../../../src/com/meesho/stages/securityScan.groovy) — `final String url = '172.31.5.29:63232/scans'`. Flagged for remediation; do not add new violations. +- **Never pass passwords on the command line** where they'll appear in `ps`. ArgoCD's login at [`deployArgoCD.groovy:494, 525`](../../../../src/com/meesho/stages/deployArgoCD.groovy) does pass `--password ${ARGO_PASSWORD}` on argv — also flagged. + +## Where the secrets actually live + +| System | Where the credential is provisioned | +|---|---| +| Jenkins credential store | `svc-devops-meesho`, `ssh-private-key`, `argoCreds`, `ringmaster-token`, `sonarToken`, `vaultToken` | +| Vault (`vault-prd.meeshogcp.in` / `vault-dev.meeshogcp.in`) | runtime service secrets, `MEESHO_NPMRC_SECRET` | +| AWS Secrets Manager | `MEESHO_NPMRC_SECRET` (alternate fetch path) | +| GCP IAM service accounts | Docker registry, GKE access — via ambient `gcloud auth` | diff --git a/docs/wiki/pages/concepts/whitelists.md b/docs/wiki/pages/concepts/whitelists.md new file mode 100644 index 0000000..9f7e01d --- /dev/null +++ b/docs/wiki/pages/concepts/whitelists.md @@ -0,0 +1,31 @@ +<!-- m-wiki: type=concept slug=whitelists topic=concepts base-sha=5399a5ddc36b generated-at=2026-05-21 sources=[code:src/com/meesho/utilities/constructParam.groovy] --> + +> Generated 2026-05-21 at base-sha 5399a5ddc36b. Type: concept. 1 source. + +# The five whitelist gates + +[`constructParam.groovy`](../../../../src/com/meesho/utilities/constructParam.groovy) runs five independent whitelist checks against `Meesho/whitelists`. Each one does a **fresh `git clone`** — there is no caching. A build that hits all five clones the whitelist repo five times. + +## The five gates + +| Gate | What it controls | Function | Read at | +|---|---|---|---| +| `skip-sonar-whitelist` | Blocks `skip_sonar=true` for Maven on prd unless repo is allowlisted | `skipSonarCheckForbidden` | [`constructParam.groovy:40-57`](../../../../src/com/meesho/utilities/constructParam.groovy) | +| `app-config-disabled` | Blocks `appConfig=false` on stg for Maven/Gradle unless allowlisted | `appConfigDisabledForbidden` | [`constructParam.groovy:62-72`](../../../../src/com/meesho/utilities/constructParam.groovy) | +| `multizone-enabled-repos` | Gates the multi-zone deploy path | `isMultizoneEnabled` | [`constructParam.groovy:29-35`](../../../../src/com/meesho/utilities/constructParam.groovy) | +| `allowedNonDevelopPrDeploymentToInt` | Allows non-`develop` PRs to deploy to `int` | `allowedNonDevelopPrDeploymentToIntRepos` | [`constructParam.groovy:77-83`](../../../../src/com/meesho/utilities/constructParam.groovy) | +| `ValidateCacConfig` | Gates CAC validation on PR build | `ValidateCacConfigForRepo` | [`constructParam.groovy:88-95`](../../../../src/com/meesho/utilities/constructParam.groovy) | + +## Why fresh-clone every time + +This is **intentional**. The whitelist is the live, authoritative source of which repos opt out of which check. By re-cloning on every call, a DevOps change to the whitelist takes effect on the **next** build in the org without needing a devops-lib release. The cost is ~5× clone latency under GitHub rate-limiting; the benefit is zero release coordination. + +## Do NOT add caching + +The single most tempting refactor in this code is to cache the clone across the five calls in a single build. Don't — the freshness guarantee is the load-bearing property. See [`docs/tribal-knowledge.md`](../../../tribal-knowledge.md) §1. If you must improve clone performance, do it inside the clone itself (shallow clone, single-branch fetch) without touching the per-call invocation pattern. + +## Where the whitelist lives + +`https://github.com/Meesho/whitelists.git` (cloned via [`gitActions.groovy`](../../../../src/com/meesho/utilities/gitActions.groovy) helpers). The repo contains one YAML per whitelist name, e.g. `skip-sonar-whitelist.yaml`, `multizone-enabled-repos.yaml`. Each is a flat list of repo names. + +To add a repo to a whitelist: open a PR on `Meesho/whitelists`, get a DevOps reviewer to approve, merge. The next pipeline run picks up the change automatically. diff --git a/docs/wiki/pages/deploy/argocd-sync.md b/docs/wiki/pages/deploy/argocd-sync.md new file mode 100644 index 0000000..f1bf165 --- /dev/null +++ b/docs/wiki/pages/deploy/argocd-sync.md @@ -0,0 +1,58 @@ +<!-- m-wiki: type=concept slug=argocd-sync topic=deploy base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# ArgoCD Sync Sequence + +The 4-step ArgoCD deploy sequence in `deployArgoCD.groovy` uses Git as the API: each step commits to a config repo, opens a PR, merges it, and then triggers ArgoCD to pick up the change. + +## Where it applies in this repo + +`src/com/meesho/stages/deployArgoCD.groovy` + +**Step 1 — `update_argo_repo`** +- Reads `deployment.yaml` for the deployable. +- Renders `argoApp.yaml` template → `devops-argo-config/applications_v2/<cluster>/<team>-<app>.yaml`. +- Commits + opens PR against `argoBranch` + merges + deletes branch. +- Sets `value_binding1['helm_values_path']` so the ArgoCD app knows where to find Helm values. + +**Step 2 — `refresh_app_of_apps`** +- Runs `argocd app sync <argoIncubator>` (the incubator app-of-apps). +- This causes ArgoCD to discover the new/updated Application manifest from Step 1. +- Uses `--http-retry-max 3 --retry-backoff-duration 1m`. + +**Step 3 — `update_helm_repo`** +- Reads `deployment.yaml` + `values_properties.yaml` from `devops-helm-charts`. +- Renders `values.yaml` (or `node-values.yaml`, `go-values.yaml`, etc.) template. +- Enforces canary for sp0/up0 prd services: `enabled=true`, `skipAnalysis=false`, `enableManualPromotion=true`. +- Commits + opens PR against `helmBranch` + merges + deletes branch. + +**Step 4 — `refresh_and_sync`** +- `argocd app get --hard-refresh <env>-<app_name>` — forces ArgoCD to re-read Helm values from Git. +- `argocd app sync <env>-<app_name>` — triggers rollout. +- Returns exit code 0 on success; any non-zero result fails the build. + +## Why this design + +Using Git PRs as the deployment mechanism means every config change is auditable in GitHub history. ArgoCD polls its source repos on a configurable interval, but `hard-refresh` forces an immediate re-read instead of waiting for the poll cycle. The app-of-apps pattern allows ArgoCD to manage thousands of Application resources without manual registration. + +Steps 2 (`refresh_app_of_apps`) and 4 (`refresh_and_sync`) are **not interchangeable**. For a first-deploy service, step 2 must run before step 4: without step 2, the ArgoCD Application object created in step 1 hasn't been discovered yet, and step 4 will target a non-existent app. The four-step order is load-bearing. + +## Related + +- [Deploy ArgoCD](../04-DEPLOY-ARGOCD.md) — broader deploy lifecycle including user input and canary enforcement +- [Dockerfile templates](../08-DOCKERFILE-TEMPLATES.md) — renderTemplate used in both argo and helm steps + +## Sources + +(no raw/ sources at bootstrap) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/deploy/ringmaster-integration.md b/docs/wiki/pages/deploy/ringmaster-integration.md new file mode 100644 index 0000000..c159f1d --- /dev/null +++ b/docs/wiki/pages/deploy/ringmaster-integration.md @@ -0,0 +1,56 @@ +<!-- m-wiki: type=concept slug=ringmaster-integration topic=deploy base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# Ringmaster / Turbo-Turtle Integration + +`deployRingmaster.run()` routes the CI/CD status callback to either Ringmaster or Turbo-Turtle based on which user triggered the build. These are two separate internal systems that track build and deployment state. + +## Where it applies in this repo + +`src/com/meesho/stages/deployRingmaster.groovy:run` + +**Routing decision:** +``` +build_user = getCause(UserIdCause).getUserId() + +if build_user == "ringmaster-bot": + POST https://ringmaster-api.meeshogcp.in/api/v1/key/cicd/cd/update + (with Authorization: <ringmaster-token> header) + payload: hot_fix, job_name, build_no, image, applications, job_status, etc. +else (turbo-turtle, or other allowed user): + POST http://turbo-turtle.meeshogcp.in/api/v1/ci/jenkins/callback + payload: repo_name, source_branch, pull_request_number, env, job_name, + sub_job_name, build_number, image_tag, build_detailed_error +``` + +For `prd`/`int` environments, the base URL is the production Ringmaster API; for `stg`/`ftr`, it uses the admin endpoint. + +The Turbo-Turtle payload is written to a temp file first to avoid shell escaping issues with JSON special characters, then passed to `curl -d @<file>`. The temp file is always deleted in a `finally` block. + +## Why this design + +Ringmaster is the primary orchestration plane for production deployments triggered by human operators via its UI. Turbo-Turtle is the automated CI/CD bot that validates and triggers deployments from PRs. Both need to know when a Jenkins build completes so they can update their state machines. + +The `ringmaster-bot` user identity is the distinguishing signal: builds triggered from Ringmaster's UI arrive in Jenkins with that user ID, while Turbo-Turtle-triggered builds arrive with the `turbo-turtle` user ID. The string `"ringmaster-bot"` is load-bearing — if Ringmaster ever renames its bot user, callbacks silently fall through to the Turbo-Turtle endpoint. Never change this string without coordinating with the Ringmaster team. + +The Turbo-Turtle JSON payload is written to a temp file (`cicd_payload_${BUILD_NUMBER}_${ts}.json`) to avoid shell escaping failures when payload fields contain single quotes, slashes, or error messages with special characters. The temp file is deleted in a `finally` block. + +## Related + +- [Notifications](../10-NOTIFICATIONS.md) — notify.groovy calls deployRingmaster +- [Architecture](../01-ARCHITECTURE.md) — allowedUsers list that includes both bot users + +## Sources + +(no raw/ sources at bootstrap) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/infra/node-pool-selection.md b/docs/wiki/pages/infra/node-pool-selection.md new file mode 100644 index 0000000..7c79630 --- /dev/null +++ b/docs/wiki/pages/infra/node-pool-selection.md @@ -0,0 +1,54 @@ +<!-- m-wiki: type=concept slug=node-pool-selection topic=infra base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# GKE Node Pool Selection + +`nodePoolSelection.run(memory_request, cpu_request, priority_v2)` computes the GKE node pool label for the deployed application (not the Jenkins build agent). The label is written to the Helm values as `nodeSelectorValue`. + +## Where it applies in this repo + +`src/com/meesho/utilities/nodePoolSelection.groovy:run` + +**Selection logic (prd only):** + +1. Parse `memory_request` (Mi or Gi) and `cpu_request` (m or whole cores) to numeric values. +2. Compute `ratio = memory_mb / cpu_millicores`. +3. For **non-critical priorities** (cp1-cp3, up1-up3, sp1-sp3): node name is `{mega|sumo}{tetra|duo}lite`. + - `cpu_req >= 2200m` → `sumo`; else `mega`. + - `ratio >= 2.5` → `tetra`; else `duo`. +4. For **critical priorities** (sp0, up0, cp0 — not explicitly listed but implied): node name is `{compact|mega|sumo}{duo|tetra|octa}`. + - `cpu_req >= 2200m` → `sumo`; `1000m-2199m` → `mega`; `<1000m` → `compact`. + - `ratio > 5.5` → `octa`; `>= 2.5` → `tetra`; else `duo`. +5. BU-specific overrides for `supply` and `demand` apply to certain `{nodevalue, ratiovalue}` combinations. + +**Non-prd environments:** +- `int` → `preprod-cost-optimized` +- `stg`/`ftr`/`dev` → `{BU}-shared` + +The result is set as `value_binding1['nodeSelectorValue']` in `deployArgoCD.update_helm_repo`. + +## Why this design + +GKE node pools are heterogeneous — some are memory-optimized, some CPU-optimized. Placing services on the wrong pool wastes resources or causes throttling. The automated selection reduces per-team cognitive load: teams declare their resource needs in `deployment.yaml` and the pipeline finds the best matching pool. + +The BU-specific overrides exist because certain BUs have limited availability of some pool types in the region — the fallback to `tetra` avoids scheduling failures on unavailable pools. + +## Related + +- [Deploy ArgoCD](../04-DEPLOY-ARGOCD.md) — nodeSelectorValue is computed in update_helm_repo +- [Infra pods](../09-INFRA-PODS.md) — build agent pod selection (separate from app node pool) + +## Sources + +(no raw/ sources at bootstrap) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/policy/cac-validation.md b/docs/wiki/pages/policy/cac-validation.md new file mode 100644 index 0000000..74de872 --- /dev/null +++ b/docs/wiki/pages/policy/cac-validation.md @@ -0,0 +1,49 @@ +<!-- m-wiki: type=concept slug=cac-validation topic=policy base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# CAC (Config-as-Code) Validation + +CAC validation runs `resources/com/meesho/validate_configs_v2.py` on PR builds for repos opted in via the `ValidateCacConfig` whitelist. It validates application YAML configs against a schema to catch misconfigurations before they reach staging. + +## Where it applies in this repo + +`src/com/meesho/utilities/constructParam.groovy:ValidateCacConfigForRepo` + +`resources/com/meesho/validate_configs_v2.py` + +**Trigger condition:** +- Build is a PR build (`env.CHANGE_ID` is set). +- `ValidateCacConfigForRepo(config.ValidateConfig, repo_name)` returns `true`. + - Either the repo is in the `ValidateCacConfig.yaml` whitelist, OR + - `config.yaml` has `ValidateConfig: true`. + +**What gets validated:** +The script reads `configs/<module>/application-*.yml` files from the service repo and checks them against the CAC schema. Validation errors fail the PR build — the commit cannot be merged until the config is corrected. + +Used primarily by Go and Maven services. Node, Python, and PHP builders do not call CAC validation. + +## Why this design + +Application config files (`application-stg.yml`, `application-prd.yml`) define Spring/Gin/etc. runtime config. A typo or wrong data type in these files doesn't fail compilation but causes a runtime crash after deployment. CAC validation catches these at PR time — when the feedback loop is cheapest. + +The validation script is bundled as a library resource (`resources/com/meesho/validate_configs_v2.py`) so it travels with the library version rather than requiring a separate checkout. + +## Related + +- [Whitelist system](whitelist-system.md) — ValidateCacConfig whitelist that gates this +- [Config policy](../06-CONFIG-POLICY.md) — constructParam context + +## Sources + +(no raw/ sources at bootstrap) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/policy/multi-tenancy.md b/docs/wiki/pages/policy/multi-tenancy.md new file mode 100644 index 0000000..5a4b233 --- /dev/null +++ b/docs/wiki/pages/policy/multi-tenancy.md @@ -0,0 +1,50 @@ +<!-- m-wiki: type=concept slug=multi-tenancy topic=policy base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# Multi-Tenancy: BU and Team Mapping + +Meesho's CI/CD pipeline partitions services by business unit (BU) and team. `buTeamMapping.groovy` translates full BU/team names to short initials used in Helm chart paths, Docker image paths, and ArgoCD namespaces. + +## Where it applies in this repo + +`src/com/meesho/utilities/buTeamMapping.groovy` + +**`get_bu_initials(bu)`** maps full BU names to short abbreviations: +- `supply` → `su`, `demand` → `de`, `central` → `ce`, `dataengg` → `da`, `datascience` → `ds`, `mcache` → `mc`, `infra` → `in` + +**`get_team_initials(team)`** maps team slug to a short initial (typically first 2-4 chars of the slug). + +These initials are used to construct: +- Helm chart paths: `devops-helm-charts/<helmChartsPath>/<buIni>/<teamIni>/<app_name>/` +- ArgoCD app names: `<env>-<teamIni>-<app_name>` (or similar) +- ArgoCD namespace: `argocd-<bu>-prd` (GCP prd) +- GCPProject: `meesho-<bu>-prd-0622` (from `constructParam.run()`) + +**`config.yaml` fields:** +- `bu`: must be one of the valid BU values (`supply`, `demand`, `central`, `dataengg`, `datascience`, `mcache`, `infra`). +- `team`: must match a known team slug in `buTeamMapping`. +- `validateBuTeam.groovy` in `src/com/meesho/stages/` validates the BU/team combination before deployment. + +## Why this design + +Multi-tenancy isolation is enforced structurally — a `supply` service's Helm values live in a separate directory from `demand`. If a service mis-declares its BU, its Helm charts and ArgoCD apps land in the wrong directory hierarchy, which causes the deploy to fail or overwrite another team's app. The initials mapping abstracts this from individual service owners. + +## Related + +- [Config policy](../06-CONFIG-POLICY.md) — bu/team are required fields in config.yaml +- [Deploy ArgoCD](../04-DEPLOY-ARGOCD.md) — buini/teamini used in helm path construction + +## Sources + +(no raw/ sources at bootstrap) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/policy/whitelist-system.md b/docs/wiki/pages/policy/whitelist-system.md new file mode 100644 index 0000000..9aa0391 --- /dev/null +++ b/docs/wiki/pages/policy/whitelist-system.md @@ -0,0 +1,50 @@ +<!-- m-wiki: type=concept slug=whitelist-system topic=policy base-sha=28f54cf7bef9 generated-at=2026-05-12T00:00:00+00:00 sources=[] --> + +> Generated 2026-05-12 at base-sha 28f54cf7bef9. Type: concept. 0 sources. + +# Whitelist System + +devops-lib enforces five policy gates at runtime by cloning `Meesho/whitelists` on GitHub and reading a YAML file. This allows DevOps to grant or revoke exceptions without changing any service code or the library itself. + +## Where it applies in this repo + +`src/com/meesho/utilities/constructParam.groovy:getWhitelistedRepos` + +**Five whitelist files:** + +| File in Meesho/whitelists | Gate method | What it controls | +|---|---|---| +| `skip-sonar-whitelist.yaml` | `skipSonarCheckForbidden()` | Allows `skip_sonar: true` in config.yaml for Maven prd builds | +| `app-config-disabled.yaml` | `appConfigDisabledForbidden()` | Allows `appConfigEnabled: false` in stg for Maven/Gradle | +| `multizone-enabled-repos.yaml` | `isMultizoneEnabled()` | Marks deployables that must go via Ringmaster (not Jenkins) | +| `allowedNonDevelopPrDeploymentToInt.yaml` | `allowedNonDevelopPrDeploymentToIntRepos()` | Allows feature-branch PRs to target `main` for int deploy | +| `ValidateCacConfig.yaml` | `ValidateCacConfigForRepo()` | Opts repo into CAC config validation during PR builds | + +Each YAML file has a `repos:` list. `getWhitelistedRepos(fileName)` clones the entire `Meesho/whitelists` repo into `whitelist/` in the workspace, reads `whitelist/<fileName>.yaml`, and returns the `repos` list as a `Set`. + +`getWhitelistedDeployable(fileName, keyName)` is a variant that reads an arbitrary key from the YAML — used for `multizone_enabled_deployables` which is a list under a non-standard key. + +## Why this design + +Inline conditionals in stage code would require PRs to `devops-lib` for every exception. The whitelist approach lets DevOps grant exceptions by merging a one-line YAML change to `Meesho/whitelists` — visible in its own audit trail, immediately effective on the next build, and independent of the library release cycle. + +The cost is a fresh `git clone` per whitelist check per build. Five checks = five clones. On a slow network or under GitHub rate limiting, this adds measurable latency. This freshness guarantee is by design — never refactor `getWhitelistedRepos` to cache the clone across calls without confirming the freshness requirement is no longer needed. + +## Related + +- [Config policy](../06-CONFIG-POLICY.md) — all five gates in context +- [CAC validation](cac-validation.md) — triggered by the ValidateCacConfig whitelist + +## Sources + +(no raw/ sources at bootstrap) + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> + +--- + +[← Wiki index](../../index.md) + +<!-- atomic: keep this page ≤600 words. New scope → new concept page that builds on this one. Do not append paragraphs here. --> diff --git a/docs/wiki/pages/security/security-overview.md b/docs/wiki/pages/security/security-overview.md new file mode 100644 index 0000000..43c4b7e --- /dev/null +++ b/docs/wiki/pages/security/security-overview.md @@ -0,0 +1,90 @@ +<!-- m-wiki: type=concept slug=security-overview topic=security base-sha=d6708eca4236 generated-at=2026-05-12T12:00:00+00:00 sources=[docs/SECURITY.md] --> + +> Generated 2026-05-12 at base-sha d6708eca4236. Type: concept. 1 source. + +# Security Overview + +devops-lib is a Jenkins Shared Library — not a web service. It has no HTTP endpoints, no user-facing auth, and no persistent storage. Security properties concern how secrets are handled, what trust boundaries exist, and what rules new code must follow. + +## Trust boundaries + +### Build trigger gate + +Every build is hard-rejected unless triggered by one of: +- `ringmaster-bot` — Ringmaster's automated trigger +- `turbo-turtle` — Turbo-Turtle's CI callback trigger +- A hardcoded `allowedUsers` list of DevOps engineer email addresses + +The gate runs at `vars/eksCICD.groovy:call` before any pipeline logic. There is no warning mode — unauthorized triggers abort immediately. + +### Supply chain boundary (highest risk) + +`devops-lib@main` is loaded via `@Library('devops-lib@main')` by every Meesho microservice on every build. **A malicious or buggy merge to `main` affects all 100+ consumer services' CI/CD pipelines.** Any change to `vars/eksCICD.groovy` or `src/com/meesho/utilities/constructParam.groovy` must be treated as Tier-1 code — these files control every build. + +### Whitelist repo boundary + +Policy exceptions (sonar skip, multizone, AppConfig, CAC) are fetched from `Meesho/whitelists` at build time via the `cicd-github-app` credential. If the whitelist repo is compromised, an attacker could grant or revoke policy exceptions for any service. See [ADR-0003](../adr/adr-index.md) for why exceptions live in a separate repo. + +## Credential handling + +All secrets are injected via Jenkins `withCredentials` — never hardcoded, never in `env.*` beyond the immediate operation: + +| Credential ID | Used for | +|---|---| +| `cicd-github-app` | Cloning `Meesho/whitelists`, `devops-argo-config`, `devops-helm-charts` | +| `svc-devops-meesho` | GitHub API, JFrog Artifactory | +| `ringmaster-token` | Ringmaster callback API | +| `argocd-{bu}-prd-creds` / `argocd-dev-creds` | ArgoCD CLI login | +| `vault-prd-token` / `vault-dev-token` | Vault secret fetch | +| `sonar-token-prod` / `sonar-token-{bu}-dev` | SonarQube analysis | + +**`set +x` guard**: `deployArgoCD.groovy:493` uses `set +x` before any `sh()` that includes a credential argument. Without this, Jenkins echoes the full shell command — including the credential value — to the build log. + +**Vault token lifecycle**: `env.VAULT_TOKEN` is set temporarily at `src/com/meesho/stages/buildNode.groovy:548–553` and cleared immediately after the Vault fetch completes. It is not stored beyond the immediate use. + +## Outbound calls + +All external calls go FROM Jenkins agents TO external services. Protocol classification: + +| Service | Protocol | Risk | +|---|---|---| +| ArgoCD | HTTPS + gRPC | Low — `set +x` guards password | +| Ringmaster | HTTPS | Low — auth via `ringmaster-token` | +| SonarQube, Vault, GitHub | HTTPS | Low | +| Turbo-Turtle, Deployment Tracker | **HTTP** (plain) | Low — internal VPC, accepted risk | +| Security scanner (`172.31.5.29:63232`) | **HTTP** (plain) | Low — internal, hardcoded IP | + +## Security rules for new code + +1. **Credentials**: Use `withCredentials` only. Never assign credential values outside a `withCredentials` block. Never interpolate credentials into log statements. +2. **Shell guard**: Use `set +x` before any `sh()` that includes a credential as an argument. +3. **Policy enforcement**: Never add inline repo-level policy exceptions — all exceptions go through `Meesho/whitelists` (see [ADR-0003](../adr/adr-index.md)). +4. **Supply chain hygiene**: Extra scrutiny for `vars/eksCICD.groovy` and `src/com/meesho/utilities/constructParam.groovy` — changes affect every Meesho microservice build. +5. **DinD image**: Must use the internal GAR-hosted DinD image, not Docker Hub `docker:N-dind`. + +## Data classification + +devops-lib handles no end-user PII. All data is build metadata (repo name, image tag, environment, team). Credential values are never logged. + +## Known security debt + +| ID | Gap | Severity | +|---|---|---| +| SEC-DL-001 | `allowedUsers` list has no expiry — stale access risk for departed engineers | Low | +| SEC-DL-002 | No `.github/CODEOWNERS` — supply chain protection depends on repo settings not auditable from source | Medium | +| SEC-DL-003 | Security scanner endpoint hardcoded as `172.31.5.29:63232` — silent failure if IP changes | Low | +| SEC-DL-004 | `env.VAULT_TOKEN` briefly in Jenkins serialized state during fetch | Low | +| SEC-DL-005 | Turbo-Turtle and Deployment Tracker callbacks over plain HTTP | Low | + +Full details: [`docs/SECURITY.md`](../../../SECURITY.md) + +## Related concepts + +- [ADR index](../adr/adr-index.md) — architectural decisions that drive security properties (ADR-0003, ADR-0004, ADR-0006) +- [Whitelist system](../policy/whitelist-system.md) — policy exception enforcement +- [Ringmaster integration](../deploy/ringmaster-integration.md) — trigger gate flow +- [Config and Policy](../06-CONFIG-POLICY.md) — constructParam and credential injection + +## Notes + +<!-- Anything below is human-owned. wiki-init never reads or modifies content under this heading. --> diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..b1b8ef56b44f16b14dc800fa8103a6d89abb526f GIT binary patch literal 48462 zcma&NV{|3jwk;gnwr$(CRk3Z`Sy9Ed?Nn^ruGlsztklcC=e7I2x9>aqJFB(1eyu-q z%|3b`eLzVT6buar3JMAc2#EOW{C^)LAZQ?YaW!FjX$1*JIcZUG1yyl%HE<x{zkaf% zt>d!6f#E+}*Jo*NafvM<-FbE0;-_L#rp}qdn%JEoAVNlEB#J^Oq`mU_#*ev4HLmc> zjXz_hFft^><#omb;Zer-%wm4hxo!wjuX3hBld<cS(JY4}=f(ZA1*v!3KKU0_kKg;R zP7BPd@Pa>g(^-RiOleKin`>KHfL3P*{k?(rji(#j2Cc0K509#>qu=-T&B!-5EBi(+ zIuTD-qfcAYgS@`Fb2^-p)4#o6A3z0&fp?~cV=CRsAeCmO4ZQ5kKgC%0el=Q&Rhd#k zaGmAbUW8uKC}-C0s~2);d{;mpsNBx9rn__66W{AhaSvJEK+c0b6ARO+l(CI7E|S5x zhaYP--@F<|99X&)9`q^2(^-Zu^Tzfm)v|gkTJHQ!G*zIg5hzoygeXZoYUEJ;iFkE# zq^r$*c|>Hmn3GapzcDYnjgSFiO^NFyTR5AH#mh%zRToMpEi(r)1$5)h455DuV}0al z!*psWuL@<Q{xTQ80G<nx7pn(8DYMv8`jGe$y`LY>Ke-2gvftfMEGf9YEi^<{B@qru zINg<ujXxYo>o+YsE&LN?)1qItJoNhISp-fZ86`XR#*6xcvM~_7=JHUX;K9*=Gu5X~ zix|O2d=&C#u_w{=B$eCpJ4L*6i7={j+{Og~`Emz@&98}6s<-p^)`0fXE4cJ<ctWt4 z5Zu1=Q&7hwbp|Q01fAs|WEuF{Y^m8tIfg%j-&vi&ZSFXEJ7>BP{>)Ltb>JwcqI>yz z0-r-SEhC@<ck5i?f%6SxQv$ai(JgV*>p)XOoh|1|XgjFaREHfsu4dAGVz*k#m+V<4 zHqvlud6=;#QWHUoTR_a8Y8+heN?M%n1@0YLiaN@GuOPNd26tik7eKulTx?mM-R!1H znB6+H{^krFXg_b{y=QeCT~qR3T4}l+b!Oz9;~|3*6F<3?#|DYYW&1RtFE)ILZ!`85 zVmvrZkLTzf31unH7Cc5E0iFShqlBE9hgEnRJH1juII*vyp&xd<d@u=M`SpNJg||y= zCC3$-kg6i&v4_PByyp6;RZ|TlMNndY*aH2Ux2F<h83n8*Lny<tFCxUr1lonS7ef!u zXv0#42T@ILT999Z=>!g`q}X_6WT6E$hhQ`Vdp9k<L%a40omjyfeqkDhYxYE{b*Bu? z7Lk=B0d(#36XNqxA7z7xswV{@XH}U-qYY9*Pz=!o-Cq29K$q*%t$nr+D4SgfK5(k( zTseuBIjWycKrs{q78h3_2<qmTFRP2G+Id2_IR1&;)*MF$Myu4LBSde;&p&n)xa?f| z4Re3VciS6ldfS6+TNA!YyTZi3QeQg?tqB&tNL_<naxCPU726(mKG<ntTtg4C%m#== zXEHHLEMa-fa*4HMj%Q+N{Ne^_!Y`)x^llL^f;#I86aEyHVQHZp5+>^<)VS?lj!cTh z7FQcQAVA@jL^cXod8cnhKG2TS9+;QU6Kq>}UOY3&TL9gXbl{Fv8@WsF=z7>X0To@$ zY@Oi1uc|MdJ$>Kn{@!g_e`-I&Tpwfg9cr>(iakDX1qciCG_1y!Di#4_)lE!bWJbrp z5aUonb6m-?tiQyR_`P#~SOu+tb_ev6JO>EbEhHK@KbeT0_FDo>dl9bMg)>xmCNB*g zG5NC8ABavuTEZVGW6jP*nAqRt3W<JjtiyBJbMsW*H<`$Ko>?7Iigc-EE~zpNJXRAE z>`~RO9$892j&I1kV;9U)xT8^}IeV`n{}QDtj2o-RBt`DGZUOO;O*lFCb<Z83{NN$r zHQmLeR<suy8vSLe+5l<R0<#ZIRYXDtS@Phgg2MB30(G~z+foAT<I*lMkX1qmI^XJY z2fIMC`Ad(_-%!Jt)!FH(!Mh=2e1T5V3|w8s6aoc`IG?qFYad()VK2Gw)|)-hOymXl zk$qX1boEs3Ig{^Zcj)Dgy`G?RaAX|Mj1|vdRimF0e&~8W8S2ntdGk4-&FHkn5d2Up z&dC?x3O)1g1Z&eDg;efGN<4RfI+r0^Q$(4ob?B#~@IZ=z`b}WtPK2A%)(G-9yUUzp zWd3`=ZVs(h=^HRAoOUd+5}DGRlc+1bw}8ztfBprLEh8a%Lsvf1(|l=d!ig}l5_})y zA55g-wIF;{r#@7(Sr};Zf$=%8%cA<K%Xf$k`Ng2c@rejM{pQ95oXhDO^Kekw#Q4n) zWKP~O{gdM5<eX;kM`-+*g@Rv4kpV}w1@Ig>_vpyGh*;95PfeGu!dyrmZ9VJ3Z*upg z6R-3Lr%_55$Hw1^{+KWx0#z`T7O6sXo1h;m?B_ur`X2bFz-Sz<tZ+SN!U0C9rXxNg z1w&3sED!r*y!(nctVhLx=rEEMvv<MJ4M4amp{v^-YTOf5DNgVP8-fM3jVp-&6>DrL zpk^@B<+I6imc@7v<c#$+wFv2jr`<@j3*ZA9{ewup^BqL+??X`qyPfS?RlY22Wi>ip za%1jMB7q@1j#<zI-F^3C<WjgsKahoD_o1-iKX!(q`0TRULj^-j#jV4}h^U*iOV%cv zt@0`N-w5h2MwKk;4%zcuL!~wfdddl&ad$P~CB^0@DjY5m?oKZA6VC^&DDO~H=Ic*5 zHj*GYBV|ko!+!H3y3qlUnh-k%x%RH-%{EAfhT|nN);ey+u%eFSJQ#lD)Zh^HNA+d> zz{u?YojZMW{5j$@h=v4iu2mTu7IzI|)Sxn!74=<CDFT6d>*J><n3Hgxs?>1a&?Xjt z2%JhSi<o7>#4huEcD9qdR9Lj4vwmfnL{%+vQ{f-KgYeqin(OPd8+(g*Uq#TLxQjD4 zLCL%ul(V&PAPlAx8D`@K8Rc`{GPecQ<)d=KWel0<!qWTMoTvxFG4Vl&B9c@+Befan zlNddgnlK#k^HgW`ITK}%{?Zc~L=yF9al-Q0u5v937@1-zKFZ;F&P;h=-*Q(Mfs^Ol z2C(TeI@N$@k8pLeb-qE=rpdSaIym;rJV}=#1qC9+=YWIQJ>ejFeeXGQ6o7<hMS&$R zfbQOL!34ax*&k2sh!M1<#EJ@>601B!!I@RY&eDriADD6wP6DcFKDLZ|lO#<k$jh(k zAK5dM>YwnrNCZ)zRJpdxX_nPZa4j#$j6v!h|6p!<MOm5dG7SN->dH}MY6#B`@%6=) z-HigguDACKBULnon^FKzazF|Y1{t(U5rUGnEU|}djVsWT-F>@@mNx?_$kF51QF4C5 zStKR$^3(fw85(4HGs9{mUTtn1)3PwxTN?6}j;32&vJ^BiPHfndLkdU5sOemXKGyCZ z@<7j(k>DNeo~QXyJkFWk!7(y1SB%nA3{v~P2c8ooKa4auM!el!Q_=;lJ$c5ADqE+^ zX8*|A99v;jWPrm(8=h;2ZAj|(vVbx~wQ{N%v;eYL<Tmh|J+vgG*f~YG9V=CQkwdNG z+lL&6E`uS94TslIuJX%@`UKzL3S#B>D_BB2LAEWCs@xauyBDl(_HIBvA(XJ7B1E;O zJYCJ8xFJh7f5sr;Y#Wp_`$4Z_H4e9bGiBp?Qu&2!@%Bl2dT5evfFO*^hLDiBu2%Jl z*WAlL5PaQ7skJa(qVysky}DQquZ8U?2@UyJ8zB#=U_E>MgE%XA$CtfL31m$rATJvC zs@!crc0=12<P+MRdxR{a(mrmpPvs<rULzRzo`jvz3Pyu0{B_eqdYN_!JshHp-ujax zmXUI%d!!vh=+m|Vd2(9rqic<hv%f7b^|^POe=}M!`pLVY!(iKfXp5&ewbzFcqe^f{ zFs%wVpEuJXFwc?ad_kIRPbMWsS`}hGTMZ(cM<hwMv=A_sD@up;;P3TkuLY>8PM=Zp zW_5Czv9))n_8Ru?{pxM2F8^r%<S6{;HzAtHP1<wS27G{@;hveVZgsyAA|Zy&-JQ|z zJA0SO<x~e)+^%mxzkSx!7HdGarW`f7;4yEr*}IpeX3zTA#V~tB?$VGrv(oOH=DByp zJU4+>*O41}RnONbSj*piG%`nyF>6ky=|;B&k8iot(J=kyoU3p<_zaAX(1ijzf*uXA zZ_5jeC{Lks+&QeFIlmzZi3+fsF4fNW^~kvC4Q*T-vrNP!<uQ)DqoaanpK*H2hQ=mu z?D3Io>x9xnen12lZQM=1_MdW76LKX(GuW`%T~dM^YX6+ras|Xy4Qhfcq=D+z-P-ea z`T;^gj3+grr3^hwqcNTJErl$z+k>{bYFm6QV%7Opth?9+>|Dn)O@`7F@=j-XSqGPW zjUAu%b3Er@;<m65)^eDag*jg@Z&}gI7N1edLdT#b)VfkTu<12Pe=eD%jlrY$!|s6f z>j1%RZxVDhI3sakg-gvTLOSV7;FV6ED=(5;UG??=WADZw^=$4AyFh#}VMe3afM^pF zF<y5`h4IKnDf$#&FS_z~)A`<Y(AUp7JHROlTP<+6+m>a}-nM8X=K?Jy02*o02@6k{ z%O!hBhjXlXKdhy3A{xGB<##e|j3^dFv~~%v2_H{t(mN7NVeS~51?D&Ozbxa`qwZ_4 z;C#Q#fL1sua%ggucgIEHZtcY=Ag&GgE|h7Q{77D!WUq`;SSGEE0pU;aoj<7-JCAvf zduN=(tx3Mb+EUXKoax|v;8b@#HJ&Q|!g4ryrl|R>WlAv?IH`bk)I24;eE4<K)92BY z6#p~=Ly9BD^fO-EL;|;n0k5yM3b(o{tq0=yyjMvU5~{wnh~Nw!2&jh=2uSJQ`bD(= z(=W;kD@aR-tEw@`OZ>NIq@SLK31LD4+w~#3iN{=<`<1R!t^$@K5>U6%W=%8_ANuR5 zs(IDuI18ftirTDARnGmF%;iz+4{MlMihJw_l!0Y)NttXC_t+s)V<<H4&g^K)rImO^ zF!k0YI|pi-0?^Ea-Fg*xZA8g%YHH?DVFW=je-1JG<2>EY>=Xin*nGX79k6vQ?beRk zy_J>@YS<aBcaXpCXFWTqo$_UB+rQ4gzaR70{T!Sv7%iNQOl{2=O>C_gMIG$yjO<P2 ztnAH%oh{t#%<NsoJx$CUU9BAK8BA=ATwMP9^74w)GN8;zxlPSA%>-y&o>S6xt<ft3 zD&p~lWx3k71V0wpIK~;q?U9EBA{_RlZ+Agol=kVUctAmSW<UJ=y90pwxGz~ef7+Ul zqxNDthcO(PFkp7gnwh)Ui50M7{-hJ%=f^<<me!3z8YiDnr?T5U2x9_%=hI9;_bRT! zj<6^ykQ%#4+W8c>f<rc<E?k`0Pj-*c=M}%i@ypj6q72z1id4~kIN@M|$nnzfj%-~g z3ei7SukMn9A6Yo{O#DWx%Tpq~s-J?P=k!3uTz73R9MZoRq7}UcRkr`_494y*Z`*XP zYe@l96AmG^G+@O`ImgY(cQuT|)*~MrA0h_&H?U8=cs-1SG|e(YfnT{Y2%Dm}L;b_s zLAh#;@Ne9|9^wG<LW=mSzTo(E{{2Hx{NqC?{=fcxGgTc$RAr=31iNqR3~3lqyXdR1 zc2*UeO)C4acOoi7S|I%|&TIU(h_)VCTlDi-uY_-qzUrhXY|H}hydS7N5kFv4Q1oY} zGJc!)aPLfgzP(>T27aSs>e|`x(f2R1bM}*518~%x>1Yct=18b&Z>GiS*>VB$+i2<f z`GpBDHfhh@3T`}W0A<q-PLrtMHC7m3Or0zYPh5Ip?wL_<-=5RE+(-xG>876zL)1cT zN33g=g|>xWE2)dds5m2+8Vy)m-u@NHOlGYxxjam21r1;xWtT0TgqKZrl}*LSkqFt4 zNTI1=3o%C*!-i;iWnlca$stRdwITA1?#fD~5OIqIQAM18BwO_u>hqL&OAANiF|8rG z_IZ9mp<Gf~rj9{>?FA-{Gq9+Ky<#NgL1gWJixfO0ziP$4T4G>vsvqC-NQh+A64F4! z-<zmo5@G6ML-Up(BP5)OHS1PlNEHOPK;w-RM+$(2^D8U1iDg6k&LXDCnai){1U=5h zu-_OZwma;oF23WmkJapx*>(t<=AbPSG%`mTl6BJtH~3RmvPhQlE-EUkEoBIP(_WMN zK~Fe!siee{M*ns1hkp5(2}vX#%u+T!Abh=<_gEx_QW?h4V@B>uOCEetEe01tl)^`V z(=cOLmuOB;8&&m%_6pcyrt83UXkJ`f9I&0KxY09}RTTs!l^_7~8$tPA%Hm#&$k0;# zF;O0zCGo0IN)X~SyKDoY1DW{Ulce|V9w=ld;U`z$t$>8U!Gu8V?_LAJAudt3eI#*! z2i9~F=kP5m>!bmb%1e~b1!1gz01Py(Yw5gOsFN#o1a&d|=PpgN(#UVreY9^99I0iG zaYE@>(C^V7pnoB~#w$2C1_TIb1N5Je&iao?S2A*TF>@vpHg`31{uk<9{zf_}s&<Nt zN@)HJxXErNflx0D8u8sq7~)CLBVf{UQW#8ySTDxc>z%dL-Fo)C$yl$%pAdqU!HJgp zh_{m1imk{<sdO#l{AJI4v)qca{LjyMguOs~VnKV0dwY}jkVO+UfcTaMTP=2z{(6w! zGjBd;P^-5fdP@Un#akt%`Wey1O4usomXA5DS|>&{ScyeuziqZHu5cto0{S}^BlXu% z0~;>_yHGd#?Kt8ErxK)z6ojj5SacQobw)-8`c!$HOI*V6eyqou{1Upm%_p!BY^t(D zDtn(oQ!jff`ddGSD;P8Hes!v)OKW-*>mS&#i0ow87;h>(=Cu0>b4)|=EegbN5=Xkh z9Ge13=3z#sk+fT<)PuUUf_%Nx@l!P?t*mni^94p^Ax6b2SVL5U>9dHH!H4DL4}@?@ z?Gpq$C**OmWliYA{5s<|EZ@QI2{-K#brFxfA~AIqq&-WSALHWQ8}%mvaNFasrtnE{ zg=sB4-RF!?)nf{>Wo~kNFgYefoFHBcSr*;iF9B!R=5Np|jv>Uf+mcarG-XGy*kP{z zISVyoPcl_9cOg-@613Qx16OGF#sH&2NTHDa_}vyidmxS~pMfY#AeQvu?AXpWNzi7A z*6&7a7!C9HRU+N{>WYTh0GXoBnXw{lQby^XShgDOw@e8TP}9Y*oFV4MVF#@Ds2A+A zXBEt3a@-IIl)TOcXx;0P;|ihR%Tq<UE3{Z94PPCuk!8lKOfT*5vC@sFx;^hSwNs60 z4EG49P~PF2^Z=KMd1@Xh|0p_}GD4w!#=kHOnr=L#{e|Hcuz$iZ-+zK(5i@fKXEO;a zXBSshH{<^;tA7J?v$~xss^%A%i|MMPyVPZMs*GU466vVLh>@DXeG5p-O{!T7Sg$s1 z8OA4iOx-!>6eK^x{jU-0SvByimK|nZik5zKIvvWVGE)4=x^&5Nx%Qgje!k3VoizaB zip#?$u(R8u{wUFC>tVR8<S9G1vDR3|uQE#C<iykYkzGl&iC(&!n?KC+%+}5D=vHUt zb-szi$)BK(%RapiKMmyz#q35r3p?ZnOL)1OSt=c<d9rRQ)L;Xk54Bli7vT)OnAZ=* zq`3bge;SS@$Q(mB=W(l!jyCi3V54;>oA%7fs?xEu(gYn>y6BB%vwPR9&RoZE%%RK! zl#Qnkl^+Y*Y4L{Xk<H!9P>(YX&aGj|zSpqO_;C3CTepA!L#4EXO|(eA`Fi+2EQ3!C zo^SpVP?{chQ3uaxu7y>w213e22cdA#l-M2kStPE%sq6vE4M*?3At!S7tIp(tQg(Ml zECjeJw8)*#LYYk_+Txv3rxsH9jJZBRrHp29yJ(^;_PEdn%#U1q`r89}38;XeF{ee& zsZEsUbJ{LtwOjU{vjL(Wvs2!Bx;<XNugqI)2@go`DrbXJfs!+z;$6H%mKo#IF|Hy$ zsh8Zs{n9hrPvg3cnNs{c0#F@wrr>#^Mzld&TjS@oo3kk=0P36MC-Ie6eHNN&{8b^s z0@jcbdejrrj!>r#Wu=3H1dgjeOI}NkhmE}K+UK&M>%7b!n&{0Zixk%^)6#@=V~IZN zxG>9kl&STQth}qScidfg58d2dF|v_U<@+V^eE@$4x;7oS3)MvWusA?9+%r<lAT?n$ zZWNoJQ$Tsw3($LDF_v58q2@)hZ9#Ka_#Ir=NQ!Quuk1|B=8Xrxsj~}Z>N>aY#eA_6 zic@S(@<Fa}73ty%b_Dmg4d~bRR^9v}(fS!I)tU3LfvZL1#_20$e;H!8t9bRFxgfCL zpQ+pZLa*k1Jio!V6dftv(ave%(lqr+^AHLe!cTXEJLD8mGj<j?(tW!^e7Wxg8Im2E zY)U>e9$9tQM-&-7>X8~#n{5G}nuOu=dSyN+b~jA;_SExZ1H9Q1A}}Rz;XtXUIOP0~ zZzS|~T+%de<AxDZ)KTtH<%>-nGI$s?wxaJoe+99v<nDsXmk1?6{!<Y1|0h8#YwzG; z|1Tl>mo%xm8o8SNEsAqAE)4LNvHc-1AX24C4k4u3vZmov^_VcxgGxapV(8)_K(^8= z2d{xCrmk(x&514Ly?e{Mf6}h3=oeP7+ZE{%B^c-kK8g0W{t<d_2neiEf6M-<x7Jr0 z>Yw3q%zty_Rd@1nbnyHMwabNp-sSyzpV4v>QsnKcQjF67%g~n&3t^<jbe+j<wO%Em zI0dPAV*ss#)RU`#H{m5TLXjt<7}b_=QC>1M<m^sLMNty2LNuZX5pQv0%%&CnaFnG~ zI+(RUGFB>esVxCzfJ5b=SOI#YfPP^^JGQw=9L1RCMFbrU{8O0LWOUdBK#j&{`tzXX zpe2_{+-8$a+o#%8MUlL4$yK`*--z&3{@Y?jP!m{g5nM+Ht=bD3o}Ok~sBQ_!^!->! z?NDVtyLXzmGYCEmjSCDK*q?Aq1;8fz9l9|z@~l{)R6GfKELc^(nV+TjjI^n0M+S0i z@YOu*Tk>|M6a0_n$(E;#^1Zgif<-CpYiMvyT+Y*9Z?&~IKSwsLa5Q#p_?FqK3lKIw zlp6H<S5W*b)++M5m`B%kCHdX0+8~>k%lio6)yq>m-`QT2Nj-q!aX7~Hlm^Xh6FNbw z<!G19;w%zLj%ZHpr^M%$R1a!ik=X3m@PnR=OA8a8%1>$#ri(Kk*GUHXORu@`aYQU@ zB~S-oIO^~abR<N?VMEx^c>Poce<xO;OQLQ5!0CuGnd%-VaHH%}%2~JtB6xH$<r@m% zPcR%u^WkIi79QG)Ep@>mkm!W73dbb!j^_xgo_@#W#6p12>w^{){VfeX?U71Xyn9&E zHa1#*!4c;?r}jv7dMN`g#&R_S215)dccDOJr=uz%LIz@zia+LIFjRakRO<v9j>r?P zQ|Xw<Of0Kn(L>0Pa8o7&W=fw17`+SqepsQ-Os5v3ncD5|N?N(AHH&`>hLY+CLOluJ z_ErpaT49zK(UcdNmQ%iA-`jS`A_1c|$W86{d_T_T2V-HH3xUqpX0QJSH%i>1i>#vK z&y{;5)^pMB=u;&_DEWakQU>j&+opIrBf~2GUh{`kG{|Z&2Z}5dwG}>Y{W_uQHaR$_ zYH%}$c`CGC-FGCetRdQ@RZ2-%ucC_|R?mHzYEnqC%u9zRBH8wx7po`=EVPMpq+hL2 zTdjVhQn$)++17^cn;<3=bxJy0Z$U;i3AqJMPJO&SuieU&0eVX?eLEEI7Av@#PV_ZQ zsa>I>B5HE996O$z6HyJfhEt^aC><@AnzeN`xs@lv>^pPFtcodrcGyqPSB?#C`Piu0 zh5=hAW|OtT9hs*G?7}@*mG_f7ae@-Nz4{qvne66kco^uD$(JbCo2ttqUm-SMy@kx% z!eDt?5>w5)M!E#C!b#Iu9GqyhUs|QoYWHtR{4espRS-LUt=viY2iygF=-j3kcU#uF z{ka2=zsOuLR}s;&PbbrB`zty&NfZpV*Y;~i*W$EH0JOGS&FMS%VK@)f*%OOrcU3P9 zq4zjhMpx}oc`PWtP!o5Bdlp=(A***TZwVwuZbuB1Pibv5uiHvW{PsE-k5IfCgUz~l z0nMeZU0R>(ajoQ0G%Il)z0BgRR*bsdz5NcqJ<)niF6|PUO0i}<4)q>6wx4K(5>Y_I z4$WMkbCOQFs(kr<OIQ-J74M`|FXIJh2(xue{hllmiw1sCl^=x_-(TtW;}gPcE>Bnl zx85i0*7%Zm(&nKNP?AQ}d~6@?D9dO%@}ouN2paSR;zyUqJuw)1SRy=g%o;g(BD|Bh ztnKV(4fcBgDJ~M@%}n-6ow3xOh<v-8B)MmZ?Fl-Vez43Ex1&leN%V4!ojORsjXsS{ zuyLrnLG)z9phtvX;a~AZRb<G&Q6!&%fU#K82k3lg6khH?U61%|uJ%iUeMg;DOJ0;D zbjlft<f}DWK&yp}cDmY~?S8<@3~Pzl)3O9^z$J#Z`w12XlKYAOD66gHRiAOl|FLl* z{)#tqXy<w_x4x!rEBd4tQup!JD4`x)>nC>C^d?PbS(9=TnO)k5p+W;pu2F4eiG7ts zJVL4M(NiZPQDy*9`H>-P0GWY#=UTnh8feiNF}hCs`8^ZDKy;XIL^9K4Ps&y^#DQSE z-<M33nZh(6dH<X4Wsz%u3&!G}SJ$mT?dDM0OUzT4)VnrWbejDhuNl@IlW!es^o`5U z-_<uX+V?~k{x1|Xbw&=6o6{K1X)b$|^T#?bVZq<nLfd^*``@{?6;qJE;jX5BCM>?J z@YOQ9NQi>ZP>^ix5K`R07kWj?`R(B?E*OyR1$Vd;8p%2Y2zEYt4CJM~gVX%MO(E1B zzXhsHn~R1ifq9~dtzuH!*3&W;r`D(Sjrc)m#EI%`Car;CMWcU0c+0r?O!)HpjEvyP zb^;pO-Bn6e-+>dS^o{q&8yEH9<utT=**Hww*!7f<!iBtNMs{62uf@--x(1SMlBkQz zX}XrZytP<Ft#CqPN&yM}i$>v}vuXX`W;NPRlwJdX|59`z?~z{pFE!^u{3k{KkJ55^ zD;F0ldy9W*`d5YP|0(E6|K%}9|D^SIq>wO)4^cJ+yCa&xl*3}hpvcQ1eP_k;@>tz= zOZnw)#fxHc81jPcTM#)jgy|0?n0(jd3IPu-lJ&Tm`#F1)o$GTwYp@dlqy-qiHFCHS zKgikMUx|%x=_%B)>n_y^+HvD2=nP`}-G_0A7)I$yc4`tXS-On8qOkNp>Q^$|Ew%Jm zYx34*(*Z3SF}xw$CA?nG9O3ZH7l)@Dp4EyH>8eXDb}AFz)k*T53iA~gRu&e15u@|% z9Rw?69nQOeJhv^^unjd-VGFwbDzf9K{i(U{xxHyM@-aI+0qP{TU0G~w+Fs>taL#Ik z4+92(Z7n%+okd478;__0GkE`&(C`k8h@?UNnM=F%A~2|TKo)q9F<5`s)KwxJRw~k; z4giS~|8AIVG;rde6I^W6m9fliR^7YT*>&x7wv^?xu(5p45n{|2F>x%?9Jq+~Tqo9# zChbeGm@9!(s;uIKae_4h@`~yIj`Tqct+-M>d>~2PCiQ?UmFUioyy&~h_DTBQ--W|q zqA^UaJMTz<nMiQ~R}{1fb-3F~h`+LYP$E59Y<2l6GJ;cUA~5b1!a-W&kMydw4#!Qp z^}^1>4tEggQ*_cQ_LA7j7bLyz8#cpGggy;YBVk!%oSdufoh5-FYAQ)v=d$Bi`G$^~ zm!O;En#M9uCykPzLZ5SHa%?hDHP5P;T4HN0L6J*r9DAvC1WWPOrd{*obfr3yJ?Kl3 z^_6dnX<Q9rNllc0B#>Roi4<$Tr!=4mhHg6ig<fmo8d=#dQ<?UUrt&|^NB=i$J&c^q z{(U|grh1`(Dvb6C0FjkNo{$$((-qcPFVQX04OI&UfXRZ21qY0|^4em$x0<>~BatHR zv%ZMJr-`8w_Jy<L+6hmRpV>FEzUSQdp0HT>|9QQG?IXj$7Rbx4E)%HauDyY!tedHP ztIbq;D)ckd-eirAHOG7icBH23*ApHA@nG*Jdh}~G?L5C^Xw^+nLWG+>hRi&(fnpY5 z?^hj4si6I{m1u^%i_yk$tco}28X8|}g5*tAEZYF37$f(+xT%XvO^`i^Ig}%cydrwF zlpL!xdO->&@q|8MiJrAxt;z2CP*a+EvV`_<cReW2<qb3~Z45BVUyz?mti1S|DM>2& z<1=p{zjhmmYVkpx#RV=#zuy&7^2Trn=H$nT{OBVF*0z|<qD^>QH!NxBF%gbqT!BEx zKB!SsSUwSo1Zr?kMM%N)@hG=&m`vRQ6QK6=oIvnUI+|C)dGKM@jNw<Va3n(qvmM!X z==-5ZQel1?700tB!6p+yK5wYNqv}PJLg}e)WN(LTm-+DWiSRT|xq5FfFu-SjEQyak zCXpU*wq$^&KsMKkd{O28Kz;G)ctGinErJru^t3G`e-~Rc@Z^D$=SS)WYKLSFG-;F; z)GV_v?+=|6Dk-J(=F8&=ZF9;b|IV%?`LBw6oduS6)k4w$lB8PN3G|s#6sZ(MfzhM( z0WCHamW3rtXsfAHTIFTLNM$2k2pLmk1ro(=uyEDS%SQ(B8ocx>qG2Xi8;YCUHYRh? zbl@DN-za)+0F9kw>Yv=ioL)01uFp7@AVEB0AH-nmB%j$RC_totFy4BKd;OPCMUMBb zu3oUUK`|{AvkM+@KPZD4Tn$(VlQi&aWV*Uf@DO|FQjLOoVw&C@z~Um*h%Ka-C=n4H z@(Lf&MDJXNS{3Hs@J)11(zo9tGp>wS^b9{Q1WN=Ktn>ZieRZS?k`gb7P4n?cl^7^* zG5-oARAG#i<*z`J0ski%;QCLD-T$AbOHq<{KxIb4=QJR<TfA51dTQK{&$q`wiPxYP z36_bFQPx3T0&N|?Ha%Fm(EGVvt|A@)^(R=hgy04Eqddw9%Gv>n@MGj=ns0WhZX+uX z=oTjz`o-VviMt1mB0W1vA*7oq1ENz{<*-EU)U;r*ODfV!G-?hdnzhM@rRZ=|qaFTN zX*t~$gc-)M7GS{#34R-n`B)eAPfebN46~61R?j^(Pg3TXR1PyQrO7Mf@xf<3VL0`4 zh(i?-SktJu8Oj?KIy4p<MKnuDMIn_k3lG&jo*R#G87f=q)WgBT^T!FZ`)N728?cTA z*Qiv%OyR*eHhC?$Jn(d@itMw3P|XIfUbwpF8~~#Ue5o_Mdy#|9caxg!9m_)u^SXN+ ztn2cZ#2?!Oeb9G4(Sr7-329zn!j7U&`;i$&!#}<Fyx8rEmN&Xyp>@%5ZH;P&p5LB8 z^}7P)9h}vUP+1Hd3nNzNcbR`%1>dSZbWhiXe-CcB+s9e)_w<{bypZ(@cQT`P@ch=d zSOPhExgI31MVFPsClEXe>$~qYQ+d}7(!BE*9y%AjQ47BMDt=#>`1ie)|ES{pFFdHa zI)CK`f3x>)DtZnm!f5=e@g;3iK^jf!RU6hpjYu^V#q0uWLuJ-6={Ua3gDi9#*P7;- z`rm*5)n{2QE{UZ01PVy@_9(amogzzOwYcVgp2>LsJ(}hKbX_!ayZ7=U{!p{BH<IAk zx!z*_*pzAR8SC(quZd67qpOxkA4ca^XwEDTvSS`|M48DE#IAGnN@$5~X9@rMLZg^? zs2ftJOS+-aD|ZKXjRRRY@4}-9X|+k0xcDe@2q6rv9OG{KO{2TQV_3;24Z>ussVj(W z2z3$zu7h$KK<%}P0YBJ+)0unV*xD&6GusXqs=M=Cl&fP@Ttzfq?>H9TW#qDId+C7? zhD;;HOxDJR4dc_xI7-b6N6nZ@bUWu<bBjDE;l*R0S%DXC+wPpdPtu4@)mBUr^XjGB z7aTQ(g^smQz&T^6MW<=*##DA;rTba+^Z=<xEE*g?$>eDk<_9Rju2I*o(i)M0&~%C^ zc)a<25M<^NrsjAccy<SCZ(bjzV+!)@mkHbGuW^dTj=ZDX0C<wxZ;2A86SoWrn(w56 z8`sN(0@%_P7x+y4PU}MTvuAi4u>dV2HJu_-1W>b;xrB~Mi@c7FrW-94$-GnKXvF7( zA68!d!gkIo8(URS{(u{zRtrF}B$9@*)KH9POqOW-B$za4Sg-A&PM*on$>$o#L7pH~ z&YW8oJX3T!!@2r4Rr6ac0ZDbtB1b5yc$5}7oZSDvGF0FWTpZ#r7@GfM^MmC-p{9Qj z_JmmlTxO(^(NHqBc$ECU$jQp^;)%xnyr$qvNTd`R@j$8JppDCGQ<K~E1P@eb^&?@= z=v<noKG0hw^aCXAc#?(%?~w<;&z}T>AHQ7?fja9McCUZ^;``VW$1+G#=<;K{_OfH- z_$fp~S3K`;j<VlDE}o7W?KkziP4Ui6V9ME{H=L%dg;VM`9Au9xT;WwWKHN{6!IP)% z0u|k{_frS%XoV7rMwK#M{Df&Q#@g^K-mHSWxv_U{ND-kL8vsXXok~JrOM_Lswj3IG zLt=lzILnfBxU_|(OCw>PNNZnkB@=DFQy3{6+Bq9nOf3~dr4q8zD_t{P4-^%<4kj!U z<c+mbSZ3hQ<JJ{%_z4CFQ%_XnB3maO#~h^k^~;YFH&~CCkd>0aj`=#@G*w?!4fpM? z8Pwb15(Ka*TtDN-2aWK>*hh{R_C}*e*vSTkHdM(ETM!JrJ=1h?(_WL}2p#QXjrKZ_ z0k_yu^;~)<G`z_?|G^K&GS2KSrUXu|8M(ubB=Bl^tZddNe!78|{`A9rmg?_%gr8SR zgtM=Ni`?{gT%aYAf6>#*r>sQP7d_4VBRvWJCzw#TxA{*hktwQI3ST{8{<Idf37Av| zbTrMS7S~QF99k??GG-jyujW`pWcyY(9qjx3=2qdzSjY?9=UlPjKH^`g-weGt4w==< z4^|KJnG9B!=g-fF2h70EbhDSn-am~W(sj-@nqVzG`8qn@%B>>3$KHJIgMGK6I!d}Q zinmfq&RLRxX8P)_@@vVr0gPu7<t3Zgy@$*_8U*6w>*)uU<%xS{|Eg;*w1}2=C&?7B zSX?OLt-gZO+<4@tLeF+K0~*|xwMD__KxWgGfsUpj)KyeCM3J-f*uxe|xk;Dlqq%1< zL(PaY@U(>Z#k!C!B45JlmE^~wHSH;r1c^kWTG9_VT~1LN6$a6Yg@k<gj<NXN1tYBL z%UgbXZ$8JVW-WQS3V{o-$^1NX`};M!53n=;QwaNy2;sv|i%U62fxA~Xfa&!&e?3t_ z2J?6c4N(ZcZYd+F5CH51?)+Sc{!t9`BQJ^nL=pMfk{|LAn?FOZ^d1wD40c@v;$7nU zs4YXlwz&B~pp12HL>NF?&b0hs+5Dw=<AAlr`C39XXC)n=KsryANZ1;SB@wH=YO>0j zR(wcEYmdfgojx+Hzu89*C}4$I7^?^vYKhF(`>=MC)VeeFR}}?j#XeLnp8OhW9%9ND zt6utD8DHnQj5@YJv+$USdN{8apQir2)Z{8_s!BABmG2O#pz5lSh|gf#CI8X4I|U4g zhQwk=VEV+j+-KNxuIk96Bi%^(Sf9}A7o$zHJ5mV~)qP))QQY&^>9}z9z9)PWpw>8T z7#NWNEt<tGE)UwcK``pFGSmgenko$iixEvi-B3bJRtk=Pf&B~T=4ah@&R-ZG{YQWE z&m#{1BE7#UZ;ZO0CaNTs{|Ff8kQRA48-`hy40KRDjW+2EI;jQ>nUoUl{DP5(lm<tq zlgFTBjsD|f$BAb}TK8Q`4Tdd$bN97;fLMT}?9SFv6%yYx_s<-!lMIV(ukE<s2YCkD zKwL560E;+6MN@(ii#8TYICcRfP4k0pr{U@qwjAC}tK!O1nlXE5Jx856<FF0t5|U)@ zPW|Qs&>y<3;tpLJ3hG|;CGB`3**uH0tf9>;7w;Aq9SRVg1FDpI5y~rY#B|eCNpAXD z9692@<O~7~C`zJUUy6{hMB6M^=*4vxUHgf$G`RIz+A5=)3T@7^ztuR4;**y;+5a3! zL^9>_%$t2^nu&4lU~(~_iVf|Cs|mXs-xKlY$-~FZB$!oDK#)JgHZCG)ySDURM=@(i zCpd{Er89|l&)(&5>L6LuWY3yC6)`jPz(Po8pY=AYIBnx3y2Qx6*sT42mpR$zwx!!< zHHCc~tbF^-bje?bo#~Q59Dmw_-VcliCn^FfI*EV)U1NkNA`6Cm=^%j`%M?1Zxa=1U zn#DPNc32&XHHfUfmPx*J+3_GA&g-_pd#wO=Q^5bdhzmm)>s@yO0q|>ROV(hkhJWf@ zqWjI#+9Wx%C+!kp&kxX|XPS5m9CBC&3r>}SwdFd#YF_W78A*CN6mFC)qzOjM);Z&v z#MjdXXMw63v*tbvY+$tDmuHNFunOlRM#qe|eV&|$98!xy{n)-=N?lrkr0_}U^sz|x zs0y);(2Dooa;(<kRMRP@YjNw84s%WyseFW>9zHzRi=I{GSVcv!6jl%ck@)>JODfR? z%aI)0HvbhzY9K7e<PwUZ9k@GU67FvL0N|#V+@9M!@l?r_T_7&*xi8f=z?5<TFydj+ zRR4*obA8^_X=*>Ysntq#JvWzj$WCuoyGoPY7;LSPfZl<iNfpJhT)dfbey_i%=SQqt zf;5wf@jB{Q8>FiWU)X?(-p}s4FXQcpIp00;%Jv;k0t@2vBu4i;rh-?{z}cHTLL9Rz zT8r(1Ws*H~EyH+adP$cGv|7HkeS9p6eOEI*`idH3twkE<Otsk^dF4tL8G5TEdsgwe z+$F>J*72|ey4JgISglGV0Vo@qe#)f-=|g%l$S&Onwl@mmdn|sjXXYaQ4MlfzjiK1* zY&hWQyc9?G2}2s1fYnQ}LXpq{!&Kr97d?=a?_xXAU0SXrZE?T+=9os2*v9%Csph*M zW{}m4+PIRmHEI;<<w8LGzZ!N*{DDU+TvkX^^BxgKf^MQ_vDz2K8?zN*9m*4{RjdQ| zbe07SWb>=c5$PMrfg#MTs);4Tb_0**o}*cimSWRcxo(;G&&NV+-?W7v*%4ACG#t5J zQP=$g-(mN*;B6s)d9JNkF0#Zz_WA>J;{=2a!IJsiqCV!YLjJ(wUJ`3b$>qcZ!HjDT z2xm;<A60+{Gbp6T&>fMSbtJ|3o~tc!jJ+U8a)vX@NcxU8y#u!Puq<dmstNN6Q^AH3 z0UkYHa7YVRXZYA^_v?e<*PZ;kX$9NL1B{1;;|=)>%R~{sps0msRFO2!GM4}786S7* zxgNmf{q@|Sdnf6_he>gEGX7Hn)uih5nL&&t4`O{?V;;bdl1U~9RAnjNmt~1UPC3mh zrR8ZtHzz1(yOYSK$OjKf;InJ+7mH$WfqI^OG3dhA+S<Q2Gye}3{V(b?{XgdR{wIs7 zqFSK&>!YmIgRv>2H78?<6A=~%E{ug^P+^b*+f=j32&Nv&Ypq?DcH&Busg^AUDE|p; z<q{Rnvbs=b<oONSeK~t7o|Ws7WVm3uer&w+!(w*SW&0sJ$>8(tQxZs1+0gUX<5~Ah zT0cGckI5%nM~d`uaMJ$o%2bt^##I0UdaQ2>-bpsP4P1Vk8r7EOSr+a!D*Z4shiKFL z35Lvs^i;#;G{%ksUUo8(Nj2DY?<X>u5->J8kqS_#{B`HqS(UkzR|K5&6XI_#FH4?$ znMXeTb$nmr1`|{n*#5H1T%vtU4-H)vrtAchme!ZG#@c+Hrf4uxx$;VU(Dr~N-ich4 zMKpdwot^bPY#kBILFgi?i3W_kV%vn2J+%R5x}TL8I?B~o#VXlmr?i=y`yJi-><;X* zPCDrsU51x;mkr+t18lP<f-@M=$Fk6V!~4UQR`5xer`}#>s=6)r^gEh2$saaA!qv_< zK<Z})DgwX+L+ZfGLKj)y_MQhquI*j-ZKKh_lN=RByo|_;B(=Ja!dhv%<=vWtD2a&< z3U5!v^XfZ?c9reyX<Ce6hD_k&9abNI=))aEjj;{vI{aoZ`YueeQaz8(z0Che@?8p1 zB!Z>QP13J}ptHaUjT_(*x+P}wfV-}57aU3rp#3AB&~e3%y}0ju#22u5@mUIT!GA{* zd%-e2DTmr#$(P6^$&N0oCgR)F9IPR~!Q!x6YI*7dx6LR6n8tj(#1~!0rofeMtT#g* zW%-p@V09>&o>iz0j66K^soJWg(o9#T(8Xx-P3<bcWUt9ew&>?;J|t~nIDSGPq(?-B zOoNnc5HZhsW(m6!J+yj~kjmjV6GKvhO>%^v5`O2I@4B$Z!~DgelYWdC4P>YfmI$TR zq`atDEhIt5ua)PS;Yz1`FX@3Na6j^uBx_rNKTmgboWGwE6O5;iQiN6Q8>ZX%ApVJS zTEf6oj=@?7klS(JaijG|(gO@dTgxB3#H)4&?+@VWkTc)dl;qK|uv;WRI*cG2`6PiF z4+svy+Bfn&Fs57Jz6i!C(w$w@VWPAbRGak~oN>3vUg|Mmk0NpfURt0*DSJ_e*Gi8I zqshW4F}L&aS8x~4*#{4vOc`gKW99cx*L^69fgPj#?++q9LidItd}<@&#E{ZGz7g|c zFX$uKJ;Qv^NpN*e&EL;l@1br8j8oxO3e`g<911L_jr~Xb0)t$x$A~dFay9(}gt4&L zyb=1<`|)_7(!^xJ14xLBGKXO3`R^_;F01<g04AK~GvHDhvi11~=$|)fH_fFatkWqN z#LlTvXNkf;s=~E)@{NBJ!VuFDsL!)Mcir2ZA`xB@nzWPEzw9Q~PvJ44!|^zv9Opa> zG70TiF<5(=pRsJYj!^XjLl_vFJOQPhN#Pkr#G0-m#xG>q)GAHjE4WFhe7Zi83;gte zdDv6+)qrgh3F0}$gPmtb9-Ff1m|xDD$6jX)Dcd5Ms-(@nKM_3)2+hfh6@Cs@-=%Z_ zIinf|ck6rN{EOadGmJ-rzvxZnAL)(mf108HL2v&m)%=a*?3CnX2ZfOQY?ha_1<yy9 zMG*>1m@UzRqlkhrVbQ@0M(tSSTerx}IH@Dn2={w$iGqU#`v}PuV7I&A9JYNP%sqMn z1bTq*Ok{V>SlVH8H*4X-lO?VzaDQzAaLvc1tTL+To)YOuj^V8mQ?)K-FT(s_!ds-O zeb$rKRR-~g^+_aiGtH6kbJ)!K^ie;ipJ8e;>iy2}73i(1RY-~!(tk2zPj;pwB4k1a zVa~7lF^EE`UH=#eb**88zBH%!WkO0S?_Zu0KpRtXN+XMsAwfT56IZI}&cs+R5N~p3 zlQH7o$(zsQQBPIRmD)i>TfdcgCSKbVVD;VCmO3l1VNbV&rWc9o>Pk>ex!)Nap%NtP z&kKIFMm@k9-HeXj2$((SmG+a-dXvl7q(7n=8)cELHf!@Le+X)=++(}pKC*dcns?>G zVa*fV{2FDIJNaK_jq)WE9MvxiTm6sI%YUn|S=oP0Z`vE#GMZa`4V5byxmv0@8@Zb~ zyBOJuTAG>Im^uIL@!ZrWJy6xL{%n;pEwY87Y^xYSfmmgRcgcEDfz4TJ#{;n|g>8(> zv$(RLnp4oD1Mj>H@ar|0RCy}E{GwvuKOf1F<qz)_i5t05SXYtJJ<M(2Ie7kk{X8-I z$Nw2LaI0;a$?ag|?9(Pexi${E3JE*nbXbb7a3iJC$a%KDMr@b?Dv@^pZ4<qi&V{oe zEll8eg~r}Ae@tF;h;&({aG(lW)EKJ4WtL%Eu|?rec0hx9CYj?TUxR3-7Wp8`0CP2p z^k~?D3SHter3d*tDmXJidu=c?n2Y=L{4n$&8XWYOcaT9~OeZnZ%5|i(FO4de$=TO` z{XR&Jz`2;1?TFUI>S}O&z-Q)MmCVEK{p~b2xFj@lTn}#s4xg7h+r;n$TZDlT2AXAv z7R^$J?R|*xL^>7HI}e>7{HszA#Y_e8=~8*3zy_J$ejuhByeI0I!w-&%MW7Q-FGMKU z8qPm&IdU3w#^#`d%Vcn&q^w;EEr|w2F@ax^`R;a@p>l`U-T%~f&^`#zG}qdSV)A<0 z^*U=#=#o&gd{o+*s#j$xf+2y^t1Wj9_h}(DNi^aK#jI}z)v1rk-H)gocbgc`wB*?$ zfg~22r!^VEN+n>U8|3{Ebe#!9k|dF8lV*9c&9H~&g|$Ymc-2O^j9w$Q^I)ldd}5zv zQkBFDS2TxDn`p}-{-`br?tUCgyfr0Wbf3QeATbp=9sN|e90U^eVOu0~VT$1A5))@C zPcwzUn7bP^Gd~hLA@8EwiklMmlc^(;uPE%tLecC-iZ$_~jNJnZYn1A%r}=VE(-LG; znh6Q+b;zKz_N7)0SH7t~u#)e>Pr194w7xp;V&CpmJw<!YoA@GZbyzC)>5j6zBO%yB zjVf*iveYaWlrE~+p8YYym=-QmTd_F!`)ATishn6(oD}hTE2AqnVPF_os`ca^ET@@Z zoo~4YJASOBn<;8#(#3G>n1E)&@JA^3LV7mK^kaJ$((~ASWup3G(%#8O%xFX8XSiN~ zUF0&gDyT`FzIjtA`<-+9RXEKbwu%RtcrG!#-aoN0aj<!lGOsMvt?{M{0R`W+hN>)i z(G|=#b_!z{o1}cIyw#n=j~Ac|NnR@<-CW$c%JFBFTi5JW0BX#4k2o2w{L0EglSN7E zFUcmFVF&U6NBA7!t`Lut>faDk>pW>Lz9BSzsqWvnI<+L#wg=zw+aeL6=70S773#Rq zG@fVM9=1ZibB`>L>hKz>rHG}`pX;dZD>I!_x~u>jsx3;0d$`Q%t7d<8^lkl8w0WZ3 z(HGiok6h^#G2EzIH}G*;!U8FW>@|C+wE+z{@e{wwWEkzUEiT0aDJo2JwZR{zcX$Bz ze2pzE&v<Y8bF>KCc6@vE*GIv1LZ=qSg~HR)Jf|ljt#^m2hZF4z|32*7{hd|u`C7{C zjG>}`{SC3Dnc~5%D4yBa!V@}xSBtQ$ZWY^qs3)9jTuIXYMgPF5E0*<vgZEgj<3421 z#qF@H-eAG@0~^sGes$#Lb96&>&A0B(=JEntcVgC%ZO4UKHyuzuSblKNHWJ}OzVpeS z?8|{P8FtkJ=~%YMf1h*@o<n-7^?+zhQ&=s`jYxG`=)e;M!SUt+K8luk{mN_ZaAj#@ z9n@RjD*O|EtT@RWKRqsx@vJrB((291p>-YsZkLVQU!43cY~nWEmBt#&Ar%7WClZK8 zSe-!M)B8((tj^wSIm3?e5oe&mQs6BAE#Y7K*^bo<LiE-5()dUH6#l>U^Z#aITL%-H zul5Gx*FKM}n~RnE*Ko3}nXrk8nTw0Ok-d?{|KMda<$n9cFHzkfb4wa&Dp0x>XjayP zg-KZ^Ayey*gb`NecHls@$a-2|Z!Xe^@P`uYYo`Q*jKzDQGPFf^GDQ5rd(-X3n)&<n zV#5O{?oMcT@#$&fQIsvgTG$;2U)&nwO1yGR*3Ps97WZU08Oy^|%G=nP!LRzb7^7## z&y_7{WKmzu+=*dQi<3YsE*EbsxtZa&j$l=cQQxfDM}5MYoDNBu-N<QT_KE)YvUkXL zE4~u5M<;j@Lr0Azg?U*!sogfmZ;9O8?8%lo2|pIdSW8h<%fWTX85vS4y7gPYXH!US z6iZUD+ky0greWKYBNM89`P+%q!L`t}{3>f|bD>?`-DktKL<0hWK!cPS>L^@|VH6## z<x|>G*0#NtGfzpZpt+e{yL@K$|Lg*JfO%I+hp&kR;NxOJ+y2H49xZA7=^RKObPZi6 zL&R70!l_{PTFcxI#h+WsO^Y<`h<D?{v9EKxg&WZSCQo~65^<X<&{e`YB8U{*yI<R9 z;kvWKv}UFp;q$R>E*z1vg9n7nG-6n0xBU8F8yDd}=?${Kl$qim3(S98@^W*vvSs{l zU}!oUIXap-i#nT`er<YL8QK1Q-y%kR%LP>(?avm4Q4-snuM&-cwu#-M{K8n;l1gP$ z3sw?`ls1z%eb%&mNBvLuEci8}-Q`|kUw6;F0-pHb?+A)+BLSn7_@my}6u%J=Ub~(* zU1n~wcfO|73IBZF;|Bhy$0FeO^>lmmZz?ZuZC8$p6<>B{Lsp-*mS05IVU00ergKWv z(LIsLS=?(>QLLQQ?bdTpyO?iiEL`;>(XJw^lA*7FCd|$g@c3VRy#tUf-Lfs*_HNs@ zZQC|>+qT`k+qP}nwz1o`ZNC1_y*J{2=fCentcZ%LwW?M`<;<BgM~)m*6oW*^(rKZ4 zW7ZjQF(v7MBF{jXInYsxiITW$qKZ>L&dcdwa@4GT@LCkltq=Xfy+OasOLT!lXrqy` zEW9YuDcfQtJ$oJ|Ln|b|q*_a|YPgCbBBfQ|5;-1(P3R`sK~3T`TtVV6yrtDbioJKI zPDV1BAaj#<dYv#48U{Q<Qb!5xY(o;uIAOQ9)x_)xWI*1k=gnvAh6PN`>O~V^ll>$# zNC?nv_r5RiH^A2t<)qzcvns9Qd$_UU$`jN;KUSNqMCQiCFCi3A$*D#(v=FXCqz$SB zyC8vjHyJhMy$5kCi}FBy0NdSCJa6{q(|*9I^zwX1NHX*dHOIDB8bsI3_{(*-k<f$_ z)05_@O7|E_p{@aX^!1ue1n%%>kQV@<?X88ekm8h968+eDGnRorXbd4+urSR~1n&$% z?0*%R!xRj}T@*LTEQl1?B_bivC_<3xV685dxk-@anB1%zoJZ~0ml#GG*vXvujM%^U zRFdc=aisjusn$O(Pfm2~TSNdOcLmva``HgG(?iNvJgxLHm{w(X4EsLGj3%y`Pbe?* zT##Jj;w$YuCDqOf5m%E6iHP-b3^cU^FhXWsqG`aLg!tASE7r2P^g-#vW<JG<gi4wH zlS|Ah+0|%ds>ng|lWd*nWx!(xQ1stGMcRDjH=YUQvY2^uCZuO%-0Jw5az*F1nW_|h zR~z5DT4j&Z7527|#z9b}pmRW}p^|OrU(TWox^&Kn>YUn%%JlZJ^16vzy|O|GnZsf3 zSXEMjOhuYZlh*ikE0&zHt5va@6&GI{1&D+NPop@Tss&f!V4;}nqX@iOvdonoDa}J_ zE-u%qrrUpYVYSGU5NeXJr?#B#3dkObD8uk*U|u*zS;T2YgAk;<A@bO^n~4;}suD0j zoNfbS!b8l~NObJm&0(FyK?k!u4CW9dx`m9&!PDWv7PM5E#plUqCM`2$#|9Gh6_G}N zN4jRX@!cZ|fx1(0cZU4%bwI?@*^x>_kdF0s4A6A*YGO4)#dKwYLQi+*i=C3N85d93 zAe#Lng7EX?@}-FPvIdp0y!`J@^1tg|IHwZ=C-i6LW7u!d>#==7<(?=6?caFCo;)AM z<q-0823IAnM(jF%*ke`%uRE}x!GP`NAPf<0tb_Q6Wcblzpm&J`@0G`D;=ut@(;fS- z$E|OJyTrtz;u9gx12U?;uJ!o%SlJ~XV{SOzH%*g<c4FLEU-JlJP>wwV6XHIU7}%D3 z75#&7SiVq=f6k4N*gy{?o~K9`+fsId8Co*62ksPHLm=SB>G)@44I(Fbs1stfE==|e z5WM)k7Hs~OwT#*$%<~0|<oyc5kqtcowvz9(fy`dYPDRZp)<%43mb6O&L@M#lc<(7o zxr+U!=LnzIh`tFgw4KZ5xSRuB_WdWO^0<8FuKeVrXa6y#^8Vj3^?#i|h0Ps*r~?K+ z-`U!zI9LhU+Wh1@KUvVfB|5?JQnm~H$iXwSjkIN2P}bw`#;g>BEb_6HV0F0=kYy;P zdAZbN(@{*9FL}4bSi-&#J^2;N`G{J?KFD@i^8BEXQq3$Q#~shvw_cx5r%ZlgHz2&Y z*cU<9UD1(G6qg=Yx{LRix``xh^Yi7@j|r7hm00t{(0ei78ZQbt`JV={$XlXvX91YH zxbI<;-YQG@9xrY>Ar~yWklR>hQ-X6TUxD-S!;~b9lu;Tu@f59S=euifnkTO2C*G;S z@TJZ5{$VG<^ThBbq_74=9q9r7DxC6VBngr@olJ}~W87-NEagn(;M*)7Oj2!(TG+}U zsLu!TV4B7DH{}gtanAHawLkpH5_$jk$0~;0`rM1Hjkl;4D-KsjXTl<*z|E`_8Nlb6 zroi&vNu(socja8wZ}9J>;D}esqgs4BR?_u7ZyELz2k%GQjtG%Vx+yeS&QI*AK1Q~e z;1-8)WjT?WqB>et(n%42u5UPI+!F^B7Hx#oW{i;??}{9#vpvk}lwvHPB$=-+pnIAL zGBd3sTO%<f+b3r1M6~06>TRGFw?`Nh>DzU#VeO7C?`w!-QT4ZgBE!WsS1clJ&i=m$ zHn^;?BNx^_wESMCsSKfxi542WFvUJUh%GpT-JP-b+D|wh`H$h4?*AT6uKyK)=>%<U zoqooUzMYJ|xy=tL^`Fs0%<ymH$68^+21x+Ek|2g<m_IWXq#7EW>&^oOXr5Al10+ld z9x<66pEk?hlV|$s!otJ~_Kz3DcB~XFzWq<@HMwvNFc2}VQu<l>S$6g{U$+nN4G0`E zua0)-H1D8k;mm6E{(!pNo<L<qJu4`v(rRJlFm>mCz*qxv$pI3NvG>(+Q4AcJvK#K8 zb9SOKS@GC!pN|JW#<}*37GFj>D1wi~_)k#-N5izNy0%(q7hMm?oL_Ju8jMFGA9bKb zv$!gbC9lC0>Unx?+*3GF(6ZZH<(4j|5-Om02Y2z2IG_&xn+2Z`6;N1An(~^lQwwUQ zOiKj)?fuj7EGlb8nv@w<x58!=m3&M~L59Iey5BP3)*BW?f?`mGEXxAB`qvM(-*cxq zxCOJFN2TAY`p+Jtg}H4>Ds4us&o=Bt%l*TAhB{h=R+Pddpm83-ms{V0T&<y(tj)Cr zwQt%k;j~Le?7<m#DQqCdnjF^?RaB)Q%TL`qPbjs8S36fVcgdq_AO(FU&rB|KDL_qy zg*Qn5*gS9X>ofYt=D7dS=Kr=V{~wzR|1=j_+3Fh+3mcp0J6k#Z&$+yVt*OJ$s$BYK zRx!5u|IH#%N;9@dV#r@$o(;Dy3GBon{2-)SK+R!>`0yL(nq~lFeelQy_)_BZt2i}m z8rSXb0|MpaMQpG<_IaUCD@=+=`KtLmC}H1)-vV;8Y!fw&`K2B6oou$<Y1duPQ=M+F z6HEZ6wWBwmf#6hsYob_eY#?SBQDGXaC((<yFF9yqu(zpPy`|*2;j4G@a))5yCQQUq zvGg~7MFq3E3NaWyTsWjWk4JM0{aG+Qc1x-ic)a{_xoAg*;Og{}6y#pN4T8X)DK^ss zA+T5I<m6)0qS4B?^<sh;&Sm*zryo)oL4jf6tEF=3rBPF61>QOj%XL`Ye$dX*5~GV? zjoCc8{4m*B_lFn=K@#mp@(*Vga>;sjA3Ds|(a_aGGbuFi)9-z>)&hY^h=PM>jvvAt z$Q7Zfbr%lPeu2OFHW3uNyavs`ezAXnB`OuCGx+U1e%!gwF?S3T3XLaG+BzOfiLB-f zLsTI!R2nT{#3)Z+EHpqiKXE$CK-~2S!*Tvgi)l{*o7SZiuHQf&N=j<zuYUv$G2Xz< zJFCvIg4hs2{@L9-qGk=+Ip-aF{w{s7(Z;Ql*PDmW10IcysG=^)2>K$gt6|+nF)`Gm z!Txq?dNfctW^}=z-436nDud8w974=Iuf~cqED93ykXqf1w8FZK9fiO>iyHhGH6`Xa zy99CYP)x3@)FSqPdVt-Br1$H%x6;EwpuBzZ?#_D^RUI0KPMzf^_Q2rPhK)0jFB8Xm zlV*;2seylEHqM|s4!E5>k-zx$17R0R2*LcwM(ea^%K>Rf92id$mc6SChy+Lhh?+zh zvO6({dx7GOFjsuW1#TIks9C3Y1NS^K;IL#Bmt5WRAnNcc>QhlO{Vj2vmon)s*asQd z33&IEDekAAXHibwHHW4Kjin6FB;UgbL))#+*%fRgj<hMuvrP$04Yz2L9TLZ(n6e~C z@W*XteIU|0ds5a95-!G3VzobMP51#RwCp3&q~a02X8T$u7Q;Zi>q!Uy)J$xt^A4P* z=wpGU$DPMXW)DL%DW!nu39E+G5tKB@YM$r<N2@T8B;FIyIU+suA!VdYe)=$~w-`Z3 z!7h!%C2TB<<dNmo-X&3^==-lMN_XeTLF$LLh4>#?rOf~PwEaIWOZ?-rZteokPG<k4 zQHJ_Xw*Macqm+O28vez~)=CD2B6Jf&TeQxr3QvJ5<wv07$KNp655#B|>ZsqWYS4;B z|0LjjIbp)2Q9#;HApIi0rAAv&MKYgXU3KhsoOYe|YT)zr{({<}EXL67@nFgE$g8n) zlwsHK7H3m?1l)9j7MVEeKIFU&$Urel=||l_I+%2%vpEWGJ4%Ae=4~9emV-GN((dey zu%{X&7)-JZ<Q7p+<NTvXv&xFm$sMByi!VFv5EPr7Ky=<7i~E<*lae7-e5k=d#}@<4 zP#$d~o_VOy7&;_~%5g07Ktqd|jhGHKSX#CfEBf-zYCF>@$2L0Yqtni7;-H%fWs%8= z=kT2S6oOA<-_q!hTShh=6tYB`my{cf^+Lx>yzS~3hAy^=8Fn4^M9<W${^Cpi1PlRD zd*weJ1_BWw^y%rdeykz;9J@32GwXK7&H2ntW3!gRCEqu1Y=0=l$Ok&zR1t9zbT_Ey zK_bU{%R}DEV+Sy!bD4IW67#O<5un;JFprK*@r==Fv71NjGtK%LGAjwCOxu~|*20zJ zNuYP-_NDpDLj@`x$k|6`v0<d!yDi&Ug^mSkr?Rvi`rr};AwA|w1V`LMt~9UR#_Aga zWS{b(fN7i@`~@?qg*INe{%)^6@0W?gd<ujE#_Q#yk6Mf&{dCt0t++n-V$y<2!)TkB zmBj}K2*iUv3K~{ca&_D(r~0UDdMAWfwr)v3--1n5CMN}Cz*KoM!zLnvl?4{MMC2Xw zF)G@HlAd>*a;F$7-pPb`5WTTi>BH<(hQt<2d>L}bEO@qeR~R5CV6M#}U~hOs$t?sI z7o&N-naKA<cnzT**y6MGnu)pkxi~kf%}JjNd7S4zb9$85O|ojf^AX;`_SNu=cE-2| zdIX(N7#cfujg+;hM;VW^YmxH!@E<@&N`EI<3s3xUB{rwnA15n3yZI%QmUi8WQ2`pY z=vVBMTettTh!$y5IyeZhdu2V;eEZun<rTS#Oui+M41$6=eJ2K|(T`$L#}DH23FYnA zKOqxujV1u}$KM40qrVaSXMg**g;9|@w3pJcGvCN)cRh7)v)`aU{I1}Q6zeWRB)&x5 zz(5^6$SOQ2=CNjbGiZFZ!!^oIW1iev@z@(w;V+r_PS&+*EkUsQu=RKsiqm$J>!$TJ z>&^XTo(>zGjv|b*XTI$ut5?7&&KtRH*Xif1`>gBEp7*Joo(B{{&6%EYr?;2euFLC6 zyxINGDCvA&Z9Ke6+p?I9Q!B<IE(s<i$qYzfw+V5KcI)5v>McUI`b0h}(?yqWH@VsM zQOR!?^5j*fLK3_B=$34i3+r{u7IgD)M~W2q7y3L-307k;BupXtBuqlRxD3=-rhwa9 z?bS^@iS*Hnd^;p2cOp}nC~VDSN?;3$3z!yI^$)`1W?UAhtCjjqn>M&ph0;8EaiL{z zu|C4KQm1Ko&6~iXk*x&^ph_a+*qDsevtmcT;T0k>1Tvc@2_|YU#phijBjGm~(FAS> zlUlF>J!lV+cX^mbgNt|q+%c)}o#I2L8tL<mCOO8G*J*hoWS|*C92Xm5pGg-&(i~G; zBTmy>)BII4PpHABevx1oqq4Fk=enLf)lPJppehz<mKsShCOFv`r!UX8&d5cW)x`ic z2AR)kce1o6k69XFTM}FF_oj!;5BRNJl$@KD#hjU8EPR+tVbXwG&ofYOGUUyZESPmb zxzR(+hf}uupQTxt)>t;iO9UQ2qK{ycJZ}25$Em8#QCj@IGeY)Ih;t1C_j5#Indn9> z?q%Mr*&t<`FGYDnXUw!Q9F(&(vc=j2NyA|}`{O%(aBk4&ic|F*CyG^zcJTh7Jbkku znj-MdZ0aPz3?=kXncCW=-<;dP;J9T1y-C;{aJj^)J(P2N6H-0wO?ZvS=U!GHKVCK< z=aWv?u%5>H&8MwXa49`eLmGW<%;nt}*#2=)K*`axE(dLvH|fGa6F34#8tRY?cr_y0 ze3Ys0rp;JgADiP65s|!r+v;Bhhv}`Vm{n>M24Hc%zOJ&UhG2A;(vSJbsM4>fU{u2_ z-6VIhEcV`qxRO<T`9g8DcjaR5DLwJXI_ed>ML_k8tm<UCBKO?h;k*odHHm?$O@2j) zB(1`4)AEqVOuJVifClF~&N}H*(h_j`b~viNNp^T6Bpy)285V7X{khU?Xir3Cu|z)P zw22(p#i5RGomgoKFTCz-v@bl%5U8<PABF$oMlrGI^l14At97K{jc<$$QtxGih?vf% za6?&)WrDsK9B-0w;2VoVquFcbgnTNhF%S^GNvNO$vqYGdBYNV4hg0B6IK2>xBr)-{ z0Nki4Ka!>@`U^UZ)eJ*+dVEKh%hU52puWKbEG44AD>zWsBPQobQCa)OTlz41wS`U5 zA(_e!#MIkQ_D?<^L@2G~TpSiQGc{2i*D?M}9=ed6<%52)rPN_&_Zz}kJyQ*xrss+n z+*}R)Uzw_8MN}8>Nin$jkrHrz;R3n*HT*JD&M9fIRS?wRHq#A#i(f4q5+z;_5Ij)k z55fi>(u^$A=GCiS!o_k6hWVWf;@9>(C^LB-^lw%JYn+7v`}UC04jw=#dbI?>PxGb< z^hYM;a|^$Xv8HwRyEFB<f-PA38{cR|Qbr%Bd5#NAA*|(pp`*CTCF{!xrC4za<s``D zgq|&$d*s&+$n{*&de@;d5gy2nj7`Houqix-1}!?5$+{)x=*BwTac9w>lC0EGDeVFD zsI=F15ChE=aHP6tL~Ao9#WHh`H@ZcicgWiJi5Wg12JkaFg6%fLuw^#2^+FGSBYJC) zcLQaBfXhJJ<Hm`*rBN_1W2hZIu<SmpGm#&YJ2p`C&?crFm<Gn&5&Hg)&di@<i{mA& zf@hhVL6y<S@3id;rxyt;&YOx4QmFSE@Qv}&yLbnOcpx~$BX^Mnsj4{m;zI=H74n@_ zcF7eg;0qq`gVCh1WuGMg-RF8R<>eIf<*h>U>kVP9*cRCfKc<$@qO~wd*)<>-)SK6P zJ@I^4#us1Hf$yt#&=?VaIkhDY^^W;!&OFd#L5S3wEK(42b#OVRSI3Yn=DLC>djb3m zOx*FMX7ymI4;B56>=L7Cv?O<I6-$Bg_btMEJJCDYj&%;l;lC-h0Z&kxZ}d3bPMcN# zK+mXN0W*6*N>pmx_j#kUAIX{b-S2c8Z$v=gOMvo?-ij^Qg7+-IsiMdRFM)v7G{O9O zb{zD!lmDA*H)}70ZFQ4xTkLM$F*jknM@CK!9fA;1rEyA1T;kT|rRhl7MQ@3Z8K3<$ zthbXo^c6w1sy3usEhrD|+wtJ{DqW>!SzzMAYG&n5P_48!FI7^!mt^UsJ=Ii%VFz|f zC`{_0n8zVxPB%8P&<rJ{+rRwc#-PV_sj#CvvGl@uG9gGQqbVAbcG_n-H8ZhtykF$y z&GL?_TJIYYUyc5pQQ;1%^oh)w#$kZVx<=^XfuDEHi0v&OBUFwMwH-oA!4v%ZP4yHK zlv5riBP`1UjI&8iRQleI*7I@L6jT3NeCg#7b(5$@cAed`NlC6KAA*2=g0g#Vgv$M~ zA8MGX2>U9wpG3=awF3lq(pY)ZY+X0iPX>u?nXvOVKqHlZ!kPr!p?==9sB_~DS`Wz) z-C{l?<rb6SR_5wnO+}vlG|otH!yO0nPXOHWo&g5ke$|5Z-j3w&Bv3|=a_gRsCZwup zx9scyE$bky-MFR!a*A2+@6e}bD9W^Rp5YAki+z_Z>ZU7>v`xhem*b=STWh<Xe$Y*T zEvR2;@SaZV!wuZKhq7gfS!z{^oxdO>ZXwe7a@WUN>CeYu(sj2^yMe+X__p(O0XKfx z%AXEQxV<am-{e+f40WUiUp?L1Z0#3Jd69*0e^q&UwtKjMyjzWhm1{gYC_fg`BgBts zm!Ix?C&@!(bAG`BPt~JPqcel`!y>FsfTzy)ozm#eCQhr*;4iF$jVCn@40VgXeH%1E z29UQ3y$aVZ3TOp-E~*g`Gz^slv`Lf|RO$MFBa@P)tKRuI=cc?XxIqzmXgmw~OWv_3 z79M~sk*g{jtNxD4ShkFGO@d3`N{)-(L`+B$P3o{T)<?mVxFf)WyVoy9v6&X8h=iN) z1ltS@>|L%BE`c71nj=koezdtBY4~a%t^5r3-m!3Kj%V`9dB?v%w?BxOI$&~!jUNWa z@o8Q~I6n%f3*aDLLYK<|4FU2X<CCV2Q#GXZjJf&b<sEjrWtOqu-~^b!pBpcUF=Nb< z!O%H%nvQ!K`$X)An>@*``7jnlDRq5+VebLwb4vJVL_1XDYFTUc;$dW3r<Mx;3qJed zjG`zq3$M+-lEM&e+y_@v{i3`Cv1X<V(>elP0}p?81NZ&{!uRJU{&9)O%uEL4Mkts~ z&T=;)Kjl_c^Tc3YX*8y9Lb`*cpyU^wFHkn{Z--k1SA~|n0bO2_YwyEVv91paW(>>D z5A?fn$`0!!94mEWTUFmE5+yocu&wZDj;aE3+jOFJ95*T%`pKWaqKNiaixt!T^#`@p z<bxh&g-SJ{CTuKnE<p+oU#KJX40ge3a9X29d{sHnH7NvaE-)>HlA$6Fj^5&7!Hb19 zHyE9zQWe<12XmH)8ID<auGk)GuwCplom<D(U%sAb3avEmlwmDM;?JqEMZ}PcPHt1= z@cB_TbYs}5il7OX(7hHi$U}9pyi~T$y06JzQZZU@@+-zibG&JQMVWMgpJ>IOtwPeM zHRd&LKn-qMRQRtyy5LYzR9#*8JDBD2K-E^^INa=#S{XA+rW5XKtg>7Nn^Of&Vhir! z+P>KycTUF|e~Hw_vAX%ap<+u9o9)jcAVaw~|4zkm<E$i=d}-Y?!#^F2L8Kg;vun>S zZa8>nl~i|D8zjQ^%<{;ZR6cbVD>%?nlBzUD&(9h}VOpBkVW!AuVW!MGuz;OfTWE_| z{yi!0mE#74$DH%4$iv357s-5PS(g3aXJUS?=I-+Jz4Y{Czu2{VMepL1!wV0l8b0k) zSH~&|HJ~YYm{WKY&gKO*WNzB=l|JE3C?T`VIh$Fi$wHFx68QWYRy%ziF%z4Zc<{>B zjkGSyv*i{+F*O@tKQ!EDM%7xw!z{Yx)~Woo$kr{Z7+t7ve;X$MoE{R-LVe22TZY;% zOIFYRqSw}4;Mcno^z?O*G8Q`&wbgNV%>E*DX{fnqK*lP#K0dvcU3endLW%GugLOH< z>Y{oG#ECe$UPvO#$t@?@GA5JFE*6oY@?+$jRxnx(BiZ8q{AuRkwymR+;{*D6-bh*) z-5<ZEZ-RhDbLT^u<I;vFzWWD{>@PC8lo`?K**Ec9*n$U>OJRjK0H$J@vnMoQZa4ti zMegzJ2oft=1Y+<S&X~#H-xE`4=E3N(LJ#SfSo&osi}}AXKyC6^IU&`DLjh}wctIz~ zZ*&%Gey;KgKU1iS`qO+sQ5W>aEG$4JE9{t_I{tH*SwKVixk$IyL|hvQq*qu&_4C6X zp>36)v+qAXl|OfXL8koN-RrhNjjA36)N;pjmTkOO>jg}c>35<Vb(0%Ex_FW)8e{f= zKL_GO2wgZo#(_xu@#acKPt}LkY<So-hVrZC^t2N7CkO1F%9h{;`eL-+IfEQCX+V!J zj7mEO#>j<2gH)fb7QYv#8VV2-AXJ1-O{Vpi$uIz3lMp3dl`?Wwpp>|6_$}|ROmbQ- z+O3VID2pdMNR%dc(_#%+-P-%bNIb5Irk&d>rOY(_mq8%P;dkWuH0mR4vhl=r?rV5g z%=n2Yz2%@f5#I6!(KxF>D%1-3IyJU|VW-!(l$}cWBQtobb>#9D+>HlD>@kp+qgiCj zU_Y+2nP+9m^gw~vIR<C=4iA8rAJ+!Xp>ygs?R~aXBZ*Vk8cFZj_&b8(pTaY{Y}cTT z*fRuKeL3=89rk16#2TNQ%KL}Ryx)%5M0MHy=A(uL9M*f_;^wBL-FO~J+@|(7I)GQF zGxu8y$fzRDE)xoI0MCR3S^FKd3Mzir$&35HZu)9V$~5*Kk^r{%vt!7ISD#%fswRS1 z7x8ugQ&u(usOPXbN5Z5URhEFc|NLc;g}f4JzVjlUxu&$T#yH-Omy4s=$~b=B<)v}= z;R7RHY}oe#TExRVjM2_)jF<VEaH<%#&B^82qk1_<T3%|+Ix$u5K46sJh^}8I1idQ3 zjMCSr(nTFKxh0QrONhwipSx*QP!+B$cDIpHCb;LMQKGe{vT&o(Hu$Bz$LhA$wd26I zv8%v7n2?Jr^!Ux{br))v%~lpZM`4e)QI>*Q3%G{)3ZZqgSTa^}wnjk_InITrx)tW> zN_A5pLZ9CogVv`5^<ji?p|CSXi2g+WGAGg6x%O^6Z@_-=yVXF~j}Ip}Co4KRG#|w| zn;g+=aUHp;+ALbtl4cHoPgjj$SgtGA!X2Z!)aR9><qx6I)IJvb=;Qal+Kx6`Ot^1; z1VMIj{}W-J=6}Z##QvpttMVf@DQ;_RENyFR>1_9Jm_n4I&Od-1kC6YSPp-Oxyt0!D zIplg&zC_?4NKvoQui_?BUY3EYOP5n0W0#hYf21a%4Fg1xeEs;w-CE2d_X6pd9A`2e zuiIRY)}Lqe0J(eXdpq{`UG}5w@h=I2qwDlnybY&n3-F)3(mWK*z~Y1=sqQ352UCF4 zQlI=T^y5Lp>gG~>1T94`()}Z4=w<|*zIWTL=+#(!PT$k6nPOoI-RVk#s?iWB=$tTc z;v`#9_oLoCy7W1j8Mn^hfr?}kDKcERb3jxH4>hafqve(?N%m6{o48;*Aj`VQb5)Ul zHK-31_Fm*+OH8EXSzh8{$7fljqN=ahTv<75(Rp-SR$Zz#EMGFOcXfT5%J^HHx8x@r zP2)nIWHes~>%OVy%4>O3(0{X?N*ukyQv5>kKb>M|32-D&p%1(V8j7s?3w|Lp63nOV z937ts^a~AioVI92W$?353}~XMK~{A}5JkKH5b=n9Ciq@IDBAB;Z!IUAV+ciiDvH*j zMD^3Dk+a${QM5$azio{#f^OHOx>LnJ+5kbRm4^N`5ii4(4>XD|b?3s1jrWv1Z}MFy zT9v+!?Ds9SiLUpcRnr?JG+C=^SKkC=BwXt~F8Tyir)=)czcAl$Z)2R5pR!H;e=OVl z8*}D=$~ONscK(|=^G~^sSitaqkw<2U?vov$hY7)fa=I8~62|7IuK10w(qZq9BnSjK zt$S9yI^QU{77(-&cteiu27n8-8*tNC&-dMPS#upD2hi$Q=J$O0#Os?xwTN{WtSzZC zp0+5nsTrDO-C3RykP7Y)6z8U{uiQ@973Pg|STBrbPO4R4VU>jA3ZJD%OK)mD`u%Bq zjUA|-$B9L<r7nkD$tDdB6=%yZa~$Db4TX8I4ie;dDzpGDv%D1Nw?LXY2||r+JOj9% zEa-B-&im=yV)i+gg!?V4uhzSZ2|kJKk+4~{RM>(11X}nY*naJ%@8ESe`WsFWU8vR= z2;2}9@)$?_zbc_riw26%Kg!e8Kd<=z-OEDxpIr0*^LqcyFQ+uzy_6rD_)MF*+Au)L zK+sV!gc8RX!}1A93BeHY86igj>{s@tCS@2Inb@Wg|3Ir$G(TxPHZ`*>y-_zsskEEv zlcqu`YL%;Yn6XuOyEIg6vQ;HLymz>grb&<jZtKO!iC=wwPIL4$(Dl4;gpozV&w&;% zq+YGJ*e=|iQ?@7W*s!3Mo0USis4c33G@0qK=MxWTPGbEPq+Hs!5LQ8?Wc5r6Vq2*I zb0t%p6BZF_L6j(MLh<I=DxE+61&vwW{?LD)bm$jFH?!+nLWn?3?16Ia4n(|!6kTS) zjeXu{(Ss?sLtRN%!_{Nc##Z>Gw(*<HB>q#A5?6USh=@|D2=%(`I*cmsk7f^9^}}P? z?OW5EW$5ivagZURMyiQ!)dSTd0?Cq6Pu{r&OKRfiuu+&nj(M|bhppFk4ze_}s<m-C zLw7r8J~1NVn6xpEg@d8CQ~-p!)e;OQN;YMnsNk>Sz1;);PvKNiaE=q^G|5w^Vy2SN zBs0<lb9OV9L}T&E#^3V@`yKdKg^X#_xf;@rB|jRwtV$H4Z;iP&(&VIK%*CYVEmn8_ zv)T@0%%MjN9Cxn7n!QFc7Fw)LvVxYNYiW(*#lZ)(!j?&;0c9YjZ@WYae;BAj(`&53 zRpwV2X^I_xC)uDc%Vq;rVAvt5Fs&|HD&8;@JH*)LYhYD0bCf1`GX9mvu`;$pTsi}t zdW({*M-$~v8G{Mg;mZ~za9}(f2y7~H>Xts91C^d0dq<=JmXesd8D;1K5UvF9<j~QU zL{k86rPwn;)QP_b^KR2w;HlEM)Ga77$V+z9j*xOPmo3w%N7NZ6h7<z<ht)Jbv0*`P zBgTd%qsz+8U`TP1hpvh%3tM8$+Co!%PjsVIPGv=f4iq#m_bNfOPm*b;5_(c?*CwT^ zGA(yU`-9jd!0Akf%?_EB&J%F^tFbO|?G&omeP7x}awvYcbgiDP^a&%?r&vmjsxCFz zDRi+AGlsCDHBhj~TRy2nE25)eq=Qav@9L@*)fpTCUuV?#io}#Gd2!Vo`T?T16bP-Y z^$%_fntg`>?WTYl6d%lJqXxN`Pj}5LxPgSRE$%)Se9Nn;^;MLmXCiH$)23AiNRlj3 zB5S`@U11=y{xj(rqgS3zSUD^dhUILAwb<rg$<BFM9*DR0?pCn81br6$68a6u;TUql zxM3uCM|T8<2zGe}3RfjzaSUTk2Mb{*d*(i)oVWuR`=CG$a#^$p9N`wO8N+}$`gwKc z@<BmfbYuuf6#Y2FMAq>|IZt>UN#gv=Rm63ig{MK*6HQPQQC{?1ODO<HoGp(FC*b@= z{Bs%q@D{ZQ!u3w@xB0ioOgDu&kDZKkynW`AJ<@g!yZbHkw(Z!am#W$T#A^u=f{!DA zUiFK}PQyoLfcWcz=agA%C$7|<VcG9sWG@-YO47>*flB7}Q(AO3hFI}(g&O+0tS_v* zssss=fjAF6c7M%h{bJFcbm>-<=R>Xa4X{qGb3|a97zk+R8pO+p(k2^QM<;%(sz0y~ zRB?%#!Lct8vXEtAzqvF2#x<l;c|~5Kbiy7nwoD5saaY6qD70Z?sv-v96*urGjm)C2 z<G(<9@E5<I`jVb2x%$gztCGACBeTouXT_fck0N?QId9Zj8T|F@qs$yqgIFzo*lm?t zFbyhzks%DS4&#LebP6UGE<JMW(2NjX)SmWCgdKf=@Qxlkq<}RP?nN{5_|r|^4!2EQ z-GCo6(U{Xc*mz--C*LL+3>o$NsieLB9TCSs^E_?X{@2BD7<@uv#vvJzQhJD^v3!dT zl|$vIA|g+p5nMz|Au5{UAyp|$2kfI)S~hhN0%yOnr(#(o-&bKg$Y+VeF{*sx3Du~N znZWwrE{QHx{GA?2J*uLTQ+AKA)Nbt+N2AXvftlF`pev3SOJ$4`MSDf=HiGkA5i0UO zd~$T7PLbVXMt2^U57wmD5}@X1U>&QO#B&jZ0J18_+exP+Z@5Me9xd0Jbq&L^e7(>X zNNZ(5fx4(0i?cEE=!j+2!b@EfJXIo&j};GwfS*019h#N=Yt|*|0J4<j0`xNu>`!D5 zN_q7;3^d-)FNmK&7&H^rwGK+yh}q{Hpt?|PFC?Fm#mlG5xknmlrQ>IgB05c3KF~=a zh6K*nAvP~CiOXlXY$wlxYQ8_)WN;>NeiQS5Mb-&Nuox?GER-8$-`li(QhmzUy}Keq zW@+_RPM`C|bx|r{2{VLpv4kQKehI>QOprT%3zknCxVb_F`5u!3W#trOn>06Z6D*XH z=M)M2!jWK4R<D6&r#H(B_6d@ab|ek2r$mVklJ~|iMDe_|N?4^_!^$`*Hz4?pisB2! z7zInKk3rmu{7O`Ykpav5%Nhoifif!^7SNobL>GLfuttE%E2P@F6hVZljI&jmjn43^ zPJ~{D)br75_H1XB8(ej-Emk3-$#Qk8x9>hEB<9vjxJQ=EG&)&*v=3TD&pvVnxeR-) z?Lb+YlOky39f%jYERz8;%h7@zQH?O%8>!r^nUZ(>IPqq+lbCHA8Ax24#IZ@dwzGe_ zNr{+ocSoD-L2*Xdg%@t^OiJbgq#@1W&4(>T_SLJKpM5HrJSQaRRfbG&uyI9+T~>My zyWR{C12~~%bhg$$vJk%xRx<*^v~v)B^3%hV33i~-tUvA5Sfb|5i=rmc9n>)2!GqKa z^P&<_F>DtK$|77CJ5xuKX-Q%!OtxP3n%EsDQrn82M%6F*?l55XtzSVcMPQG0ZuQjl zmq*Ic&aackwk$S6PqbQ!TT;VJDSX~x&h0RoXfrD8&a{@qUZfVn6$ilU9V(GVzCpk^ zP$Zf;Ui%dnVGK2<VMHlHFPc3|w0;Qatb}r@Z02ykUiNzO8Ug<Gg#zJ+MpoV{`ny4| zKoz#|7OwPg;sFSwmbu@5SKG8qR0@+sG)iNONjD5C&sE-3S`e}5`2oYsO~w)z4pGrf za4z;18aFA-EZE}p9VgO@X}YJfEZJCC-ZeQ}UbI-9tJi0j_IAiI)uZhuTS>;ueF6kZ zFhW{mY7j^Tftei%ow<nGt?(;`-*7f;0SKE`_AHdoLj%BJOj}vGkPlz&^jkQ(1A%Lf zu|TrCPwc}5#Jvh8*&yYdQ553j&QB5g?sXL&nlYw*DkfWQtfkeQn5`{rt@X}rF2nMg zVA=x9gb@eo^Hf{x+<rc;#GWZZ44g|iR}zpbc2<bmf$P)PVpOnm>FtP`AO&4M?tOT( z;Htw$hS6rDA9#f<0l{2DA~U)NOfScqg!^m^q#5Caibizsnh)<liOgU0!l)9S5_3*W z3ZWq@cuB9gU+B}!g}*61{>JfGIIAiSiC=S%J|_X-AWeS|ich7A5v3!>zaS0qG@+}6 zF+61ADkXR}zFbZ1mX?PdOp=@C9DI^|;2Tz^0qedK3>_4z?WYMY85qL(rt=Zq14q`G zmX)L~hGa0K_F1zeK5O`YjY<oeq$4Th>kt&x-#C=rX%}-v%xC}Z95zssU#Mk{YR8Je z@U4Wha=tl!xo6aPg=VsfWT-Uw*s!bATd!Jrcam6JES#?b>09?3j3HtW9zjdZo{@vm z;Qsw!K~TU*LK!uvRJbS;OkNH2Wt%Y^x3I4&v!zodO!!r6#`%hm7yl~tBXG|sE%(t= zztYj^vC$ivB^+7S$l7s@do8-L_omu&g;hi4Q7^#p%DB);DAqKLC_yf{M--fbVCW4Q zpLSAJpyR=Jw|FpZ7!OY9&`o&H;FE5C-006%H7z?V^+c?EUl19l4m+%pxM%W-d$e~- zt(|&Ex@CFK^ihfbnmM|@OUuO<vjSm=9aiCC#r~2Zy7YzWU>+x=YOaa6Up`MZSv=z+ zj&v;Xfs>|(JoZyyf*n#2H&qEvkEBqz1th01TIY?cy1siJEZd%up<S&(dvi{QgF+g9 z&1*GqGVL*W!Ttz;*{e&{6R~941)>f04|88q_e^UcqIJI$qO^tX{0Q=;ytn*d0;d>W zpbMg2hvsXQ_P18QOkwPq?4dM+V|(uRBPZ<<$bpw08v0vS$9$VUpbm=Fv(IMqMe~ij zM>0rOq>iZM<PU$(t>oC}d%y?jB;97(AMLyv&6Zzi(5LIvB?<#Ywf0)mZ_~Rdangdl z&@8jcCHuwoEo63_;{rqY2HFx=n@Y<ycG)3-pJSwbiMnjHnh3u3z1Lz)dDu_T3NN{b zME)VlnrXM6b7?K-u%C18Cu3xgCMJWPQPuY+SssTJ`bgYo)yUXFqpFCIfamWCO~RS% zQ+XRJMTyMwp-jeIdx^aI#)(tW!`Ju=r+)lwWtf}?Jzqa*E0(icLn|P^Ey>ZylX9a} zl&P9Yv{)Lgc|b3Q1o2l|S<uDHOfh0-m!C7!7g*L3^OU%fd-_qyX{8^Xj8vwIc{yI; zS-O7v4Lu9vYsly-gP@Fmb2>ANshLidoYfmF5?I`bsF`E$9kGP};}K?$qva#L^~CH` z!TFGfb4WF(Bq_ENC#V_OREgx>tR!Qa(Jg2?b%7g;M5AE-&>&(JHfZkcmN2s<qumfH zt|>4eJeN!nCrcl9Way`gTk=o|nGo|BD1pGHLvB0ih$H-WM^@K##RBrgEQ`4$CSNzg z8QjInTy|bpvXE2PqeM9*$mGv<oDuh;zUemhOW65v2ddY-XITbKL@f?%s=UOiK+~Q8 zCm$yS?Y617*m<A+U=t8L$Tzr!p$iYMFg(e5fkcV0%<5gVQX#N-#Rby8SW_<Py|c_? zbDTAX?%7haflk3FWVSI6LVs5CT<sLo=y@rARPsH1%x~OFzNy7(U}K=nkYHX@B+kXB z$L4P`fzD<R08hRp*UMp_X;kwMBgR0r*txz8{1a8l6%QrL^AB+Tjo{+8XW};gMi}7E z63zCtQ85ea&vV)UJq5t)A*D`1qQGW&(H{O_qhMTxE~^VxxP=-4sM#ZK8K+b8HslEM zB%F&?wC507z}4~7jLD2J%3|rJzY~05%N8!M3+por!SKkk?e)7l`z&-!=vq1onLD0x z*iY({*J;!s6(%m*D}T9URd>Z!Ps7Fn?$@*V_0OIlsGq$7xq#m0A&oC)8WX5OB{I{& z&m4D92ULj=J&5P>4A>lRn(KPS@|aiq-&TfHnOC`uYpkgbZ!za!sgrKX&HmC&DR$Qw znLUwmqe#(ab!;OBsne)NG--Cm><A4qT~55cgJFYpR0>qV#<+25uf(vCy<cOrk!hG= z-(}~xIjjG5kAg!h)!PPVxl^%^@7j7(YaRPR>t?AGIMoJse#4t}n3bFn42(girok)X zsLlF0m3f3uPV@^Vj<eq)H`0@F!*kKCQ<Ix_S`&O4`zXlAr){ShupeNAgr(QC(>N3J zs7vW$dREOUH=t;vnxK-_6qp*ejG&zM*m*>v9wu&xniWe@+eJ-67VZtoVET-b0X5{6 zr(c*Y=7z@KB`=B#zMR8)M_(&sn@t?LtNkyD`lrk0nJapT+`Ued`PVEyOY{v7f2Alh zxP{mY>C3kmqt~@Sx9=weAH3PUD&9e;-4Z?DM%u2JrA~7?nOo3Fg!@?ilHRb~Q9Vh0 zS~k)vttP$Xy9A>{?$-j{oKIM^!~^qOk9nFfO9U;uX<{Z}MGPU&T0}pPw4d7EHF*^c z(1Qo888T#p5hW(|Q-(yg#r6vVzhg0gpd>56bb9oH0wu}%3M)p2fxFLEy>QG4R_-h8 zU+Al?!eBv?3%sHzLA?4>j0E@<IVV&Ask#SP6%4GDTXN8Ql&Ka7)yH<nvQoOY23?!z z1$T00v#%RZhUT+TK`8|RwYaDyKpQN|a~rcdxX=*-@o?<M6xjk#g;HKwrW2BMiJDoA zVnBy<n|#IvYvu_juA~gzqSazHv*=UnbjxfR+Iz2bQ(fWvsiwwWF9sUsnmned4zoB8 zVVnCVr(}0LmDHF8gZrGadB%w8X6jr=YL&*pJ(8G1!SZ@-!N18>%7$S|RYf_S$ylY+ z4n%*ot_mG#p83HvVERPUjJRH!Ay-9T%yQe2biJr+b%|?XeE(`??bZyWEqp{h5`F<$ z|26&q>X&o<q8#jP0WSk0rvqDK{6O6Dg*W4pWwMM>$0crC>TI-zNN~}*w7-kFnefLs z2fQs{{%-wM-9ryBgJ*Iuv&{5yuKy+Eoc^si>??Jju|gyAn_Uf`ajXB1%g`EBtwiQ1 zx^awk%lc*V?-yf2mx&<2oHk?3d{TaxpMu&Sc>d+t2h>+*DNg;iw%P+Pbq56MHt1{8 zuC!j;1YlpBL2hXi-rks7|L=db0Mz7?nWiEF08stMZRP$Sn6!kAqm#as74d%`|J5u1 zZ`hY{-1<LTu7!N7H)*O_p6?EKsN5_%--1Mhvjvco2L$XGZ^*XHUHBVm(fw=FwO|O{ z7XaQwb}bYHjM2%D*CRRcEk50C04sgfow{W%xT{S|j4j1^C)bVIL)H!YD5fIg>iNNl z1=2bj@r1^~3~TeQTAAId%fY2ha|!FRU6VMpiAkkk@VViqVwhBxz8SBI0v70InyyD6 z3Bn|Jj3nVomoatTh{xa7jx;yvi_UnW_#l*M<|9E)rOc4j#iVycL>cKHTtp3#k-nKL z+7?|mS#aSINetxl?nE8)%Zy<k4d+-;_lzN{*OnjTQLDClg)^m#bRVu7W)ekQA+lqd z7z_%0ug-0w5$lXBoh67#*U*!6pDZtMbb3&U$;os$?5U->k>!C1k<kVaEZ``SD1Q@s zcK~}3EjAKgO~G0X!YefU>`<{`huyPwZD2`YbK4!99|Okznl56^r1}88nU&cpyn*~f zRP2FGaX0@#FpvKuii!WfqnQ6~#DBA2l_uoxjK6W&?wmdns)%IKg2?m;9KE4d3H+J4 z{P-@21_oU4WQ76zv4`7rf2c8VBqkLlTWX8sn;VP7*r9$|Zvr<123Vyh&st-dNnOt) zxtL4AjW-w3bde9fPrdt&)f0toUJ2&UdD?Duy5Ap7dEF=0V82i93p+Kxkri{*^!QAa z`)V#?MO?Egc}EaN?0rV`N9>*U<q0z;OBE`~1fKFDePS?~57(1|fg7wAzxZKnXHCJG zpozR0j#XmlGB;M3)3Yun!ebP-c@1gQsq9QuQchpbkR_aq!Vab<LrPhw5gMnb30t6f zJg25nFA08^A2p<_u{@@EqIRtcO&**}95dNF3K5z!(ukY7G+m4jzQvnNY)M<0jZQW? z71p<@Gn&UHl)as;WUEUtCa?XiZ+o_2%4Qm!I*=`Py<}vPCUaFy-7W)HV{(6+_f#5l zs*EXXM%?H0M6YBDF=+E|TCL<*$w2OlpuE-7o(`0Np7g#o;>}noU~6E-WouZiR;Mgh z;i}OVBurvrDpRj7!i%ICbMj)VT&(w5JB7dEWs8$MSfbZaa1D^jw$rlh41JSI!*+g5 zc`HjldKt~dEdKiq-t`OW#SHiFi#h4kU3|pR`<RgKYjdw0Z+X0J$;+4=)4_U~^!*Nb z*w?Oem=$fp$y9`|jYNr~bjBIy<0{!aP5tEAG^E_g*pJ##bxkARx_k0UqfRI0(nHDw zqZz`5<<&sCM%VYmM`KX(n;AnE@7Q5wFig22)QRp%ij}z>S;CF5SvpIp|Cl8#>|qEO zL6o_yj`uN0$wSqXQfj)_qWIKrnS3$j-u8y`GrF8k5xy*m3E_xC>4xG+3@28lsi2dl zG->G?bNPxG)$u+RlKOK*4722EnD<j`OCba+`lV1T_|o%Cj}+C3;hCa)BVKIlBcq{T z?S5n8M#~ddAB&<=6=pO|OgheatnQ^X7n6df)Kl3AXHprf@|0Q7Lknfv)NxQliUYqJ zoFr3Mk7nPzh0IdD^Rzh^bHpEmC%x<3XgRPoRrj!Lo@PBV<$O7r&;uoA%DBEx=xS<H z_S-(32&^9zc%138ZuuTi0$CYod<?f2)QTs>vKFTfCP}MVn#i1AP7T_HVVXeMTs4JO zpT_!OPG@)cEQ+es9a7Q~8ZJxuwg`RN6PqI_ZGrR{=g#vc28nWQy+I8dcb5dFR^-u; z&&P%sTVJJ;F`R;9s*$hDbF31St>mkHWdp=P*}5fF!x?lQhPw$TMi}e=#xDm^PWJok zBklIX+F!cN8)z!@No~Er@9ywmEwj?-&7I}xh?Aw0SPtK(3EQ+5LHqwwu+}k1p;#vH zrvh`dw3QgL-<qJTG|v$-#-N|Fk$gL`2pGQ7)(l{HQjWx+C@lxBdI)5b5i%IIOg)mr z-R3qbW`zq=ff+{~5HwdFK?}TVouhEH6|*4E33M=lv9u|zXq-RHS_q@F0Vgtc$X(N2 zJ1*d1*4Fl73>4@kIQ!Av--?{@#~s8|+dQ;(;Mo#ndpY6spn{3TJBv8{Ee0%vgX2)N zCCV1=Y(p9TH+hpYR^mG9QF6nF>tHb9wDPpXRlL7F+QvVV*IK(W=+D|wiR-*I;elS7 zY`O=x^{a5b-2CDtug6c%+y!Jb>;Y$1|5k+KbP-$ndnLz+PK~0IJ6_kenCmP!NG!nT z0oX@l4sD#DBU$@kjnc{sh4baeOf!mqY{x0?+@X-P%tFTkGt+fK8Xnl}SW!g#bX7&^ z+2;eo?q}&im*rirs}E*eubvzp8ZZ##(eDL0O^$sfaX!0;rmj^d#vG<0v5$vbadqkM z;c@S>jXq)Rz%lvuo_XtEk0U!0-X%0LG%_Oo&y;sC!y!Vzbv!1e%gjo7+E(!P5CXQg zglw~&%zv|GAITU4^EUXYL*ba5L|+fG{n2f#<$P`;XXQzw!rFG>1xIQtjYXPCx$0Tg z_y1H9*k8*NMu;cG(T9I5k|_z+!6-KvLctWLG?awCF`Wto6>5{_B*kX_J!#TlRfW|Q zTxT2;H#0}=YR;55U1N;<s}p`Il8ZBEZ)a{{%^eC%aSpo*oI+Y3rfr6wf;BFxd11Fn z9k&j4g%sN*l3#*?(R5Y>$dTp5H%;k}GCmbbyfA00QK5!SnK;wWT_=y7G3YX(F_2ej zekKG-;-FFYlnsInfBS-ue-l(=JyzlnCV;dv+bFa!pd>$1xZyr37BgGGzr|0+^O~0j z15^}t&e-E6dU|#)QNVmuka5beLq1^$=n5hx6Mg@fLV!rjf(f07zjUyE!{MRr^$O81 z9c&-SdtEZ{pn(T}h6ZnUS7wPMBn?d!5HMe!BHRBbb<RP<U{o+ePE}rUeP@E#BfL8R zz^vFI{xvNi;b#9$*=Kgrj4WVDZecm-`#b@a(vaaKqqVj4uzeoG(R&^{NO|Ml7TWF| z)f-%a#`tPt6<CQ#1d@qcXdCJ2nurdkCF@b%aU~Z|&~Px+h97|MQ+=;L=0sz##vB?v z3}1_hp$yUxbuym)9nbG@RlO&gjxgf%;adSDD1K%^&g&uPK}iDxVwtBEs};iYvkj%! z$G=pI*;T8%N3VbU$g7%~?-MFn(1H5esQm_SCi*ZScf014L?Y&!1Y9$1#KCX-?L_1a z2eD}w#kjn2(OI?ms}bY%`2qYkns4tn6^+?!&d#`}zl5EMZyfWx>05=@24O?2h_`+1 zSkky=Y6p<;hK&MFs_UV3Pi4-<N>ZFlQ5qOdAaJ4>=1O04Q<~*!bCF?FPS~o{er4?b z@BAktYAQF=_~SF#TF%vAsN~HdgBetV+7Sn}tl<@KS7SOg0f&fC(;da%oL1YWSL+*m zGM#5P_te#*^#`lcd2E#Bzrd<*Ozyihcs6GM{UIN@;iOnS-MRs~qr?3IfIIow<-ibm z1axfeXk3WdOtrvL9~RrkL@RPE27<BgZp*wd4XkOqnoGJ03PzT!>Wm{vO5xg<s&44Z z^_H8fu+mlCTc_*Y*6n2b=Q|}$0GIZ_Y>=Y{Si6xRMyB}nHWVL(7VUs(tiyCf+=eFX z^v*e{k1Tj6MkZdZ0LiaYY^zFpCUo+Dxx=bBlNeU*IS#VeeOAzI)Vt^$zh$j^EZMHM z**h+Kz~xZ6N@mz<ElpZUOF5-+6iM&0cy*^7*C)22*{*ucy**Xo+61x}vO|U-4Kdd= zurJHaO*@o9(}=JN!q}9XT^vVLcFU;d3@sZ!I;$-_?+lEs{I=Dz8XK(Aav)Xu2;wlg z>-#ETTbxO`K|Nr-N;@=2jQ#7ZgkFx(W;GWygjB|Jx@jU+qS`t!IrL_@Mh#X_TZx%@ z^4p_*L+-*ol_B<O9ilY2-blifvx|#QIS)O<W^ZYi;2saFF`4^z<85DT-EhnZuAdmj z8AI=CpZq%Y@Q4b4at~d<z01I=aT}g8d*yrP^;Fpp5hK_b3{v~NEH5s{^-jg{5BlR1 z49<y>w(5gpCY^}j0q<D2_}ui7kylHo0$Jv2_B8O*=?La!Kpp{)^`1Y$V$O8X!Pt!8 z6t1i|DBz%~r=T;8tgMd^7MK+-Gy2?SLU_@<7E<7n(wK-3?w+Mr=22wCL63$t8vCD~ zcj3B?;ML|Gh=sr<q=I3^&X(5hgZ&qN>LkVl4eKqJivQEuSwK~_wQU=a?(Pr}B&EB% zySux)K|s1&x?55}O1is2>5>k~O$h(?yyyFj*W>Z~9|mI&_Fz2Mnsd!nbFSyU4NmP* zk_r34gxePNOJ$h6c<xP!P7>ykvyCw$qW0>}3|r&9U*AFcQWu@^Z90;YM#zVCO^+rx zNH@pXoqevqr|SqP@$wvXr8J@&d_JP>=uXmMSW8G@sN0shx}NXhJ^U;k3^P3*Y9*{X zT_){Q>`WUL%w79gi?=u4Dq=QB^rnC>Qexc!1mCKET58qi_4>ylhJterN@VVP&{9R} zf`VGjgzL=<92XlYXsi4V{!C1%tpasaKFas6LJV)K-=vfm;P_v(pq!FX4Y?&YsVKhO zR%%faHzRDbQ!M3E;64T2WnRzcuczPxKYjJ4E?oK+r6|}!&xa}zY4)CB2A?|sZ9Z0a z|7}5bo3I!eu5ax<rIAxA2~x)eTKI12!a#fHR;>h5J}j*49lzaa_Zc8rw3g>pd<c<& z_S11p{E(R-Lc;o`abL2tbEAMFyHDdec&1V5cq7{0{H<rgal$LI_7==l_~IVnnrT6W zgk~g2d<=z4HN@?mkBCGc(4;=e2=Jeia|L8Jtas>b(cSDK@($H8DyJ~4-_*`cwZ$s? ze5h6-?o%Yb`5-tXa|0?FF6Y2tk6?PhbB~VSfa6cTW01)6;9^4dE+jka44m<(+qOx| zS7+%A4{cV1vYAlL_6DE@7TAVxXLfPEJy)0APHnPc=nL6sYxCkc(#=FY#J=VU)@bgA z0_~_L;7&Dz1PtGWxfn&<4}M<OxI+Feqml=}caNSo-IJe{@^)617|=b=2N9Yh`+!59 zj)Pw5A0Lfnq(=5^oNq_Z9R<S=riVmxi^+75M;9G>a<KiyIq=3GGJlOexPUbE?A)$* z-M5>94p>_udw=f*7k4kv58VQ0lC!J^kehlmGtWV4Mi6UiYHz1L*lE`k@;g5_yK$-= zZtu<-NFGqxlm4JpB#T7g%Ex-iNmQO!&y7g$cHfwbO|=&7md}4l4Mn9|n24rEQ^>Ux zYO+gTedMAD(2~_1Q6k*FOpy38A*yn7gLcbXj?+s+U;2tl$BG4xn$@hHmfNzSfuA*V zDR8OI{FbT?yi6r34Q}@hSTAGKo2ggB19<tIF#frM6{`4%8=5I*k<yr5UXBCqHPU)5 zvtzjX#&8KI+Cg8O3H$BN1#&fA%G5IRFd1pHp)t3~HZBre?3IgeX29yj%gU#mrBuAU z!{kQQq%M!bE?vkKi}W)6&4b1`y~Sa0GN2Q_phmnYq224u$2C}YZqPW+6l2k#+rwH! za`T-H@XVUtA2CdR>-#DmV2x|Zadz2|rHCQV8f=qYq3S-XQKr)V!L{fbjC(JB{i1oZ ziF#JsGKmxT>@0|5a3}*}b2#dWUIr!i`8n>4;r7E*)&qvB!SvEbZkC%_T$i>HF_iTK znSw(apn9nYdcK)KaXd!E__$?es}T}>(H*ztldjGo3~FxJOQHIwDEbA;V7L2u0y+iR zI<w?Wniwl(Z}RufN-xW_I$qw}=C8izjM2jPdatfSlA%TZ-fOWt_#+iLzwB!47+|&# ze9p#_Yg)o(nwO)b<W!q=FJutm)~qGt&@n_#hdAa?*5mP_BvX3lp~`nSokq@gOo<^> z`Ta|+1SVzj1fro-ACvhOxw!`lkeVnt+5zUv+2Q>l6W3DEHS!?GkLeUc=jF=*DYi;4 zgAmXvqwt<I8uguX2w)PspX!rPKh6<TS4J09D%QS%oL&B;n~6UYD~q^~n=@}X^&}we z-mp@kA>L98S&@oBP*(OL2;6Q!{jJ!x!SIzc(UKP=n25KVnzea3MJKb=3u8Cm>iLlc zo>?@$-95+WQf~)EAZt_5R=Kx&-+eesXf5(h%iWVsgV-k<5sR4Bt?SzA!_Si!Vs17{ z{6tvfF)5Sptk|88Zta~Yi^wNgFB3D>72<4rA$j}O^elvaJgTjo4ShF~YmiNpHeGbr zyKXGp)-!&Ibd!z^zbI+4QbF?)fGbwcwDyLFza9Z}=ghoEC1>_-5DRf*_-4`0`D_3% z-j$9^NUELnMfu<!A7@e~x-W|Z54UeOPbQ?4X$T>|?&hgFGHu3n@;Oi!chfyGFC1tj zysM2L<;pVB<elisKl()(YmPb_lUG_T+>&eZILeivP-DG6^E!_0P@Pv$*0)yMcNP8S ztipdgy#t~iDVyOeruzZb?;xzt0NZ53utk9^3ZvN}(iFQco`XI5+!2~Bt*g7s$UI9V zqTk}E=N|5KTZK~u!6+3ngR++0rc2UcL~b2^1ySOpH^5EkBa;19dk^IoLT_D(^eYV? zh)u!~KjQmm97L8GO!T6q$6zM-+4)P@I(QCal||#8B$YWzh+EnD6~{;lGD;KM(2Z~x zbfm^>#(c>3<zCzB*;cTeaL!i#D;P(-(kIcP8Tl7k{aU~`4&6Ek><Mx{6}`NB^MTp- z3$_nEE(5GH7L68$vud_Rj@LUQ1ZRQ;-+a7~hl=WfTXpmyZ^QNyOxlSji*OfvvO*(7 z%P0<<xSknNJeBq_Ev05-LITAum@&c*=s~0cI|^lU`;oE524?=e)Ar!3sdDvouy;D* z#a6g|#S><`9QS(Mb$0_NoT37Om8`p*ft5u4+)-eY&scXqIdG8ph(=r%k3w~PVLOXd zvY%SJgzTUS)}20bSmIE#Ku2ArE#^+hFkz~5s)Jq}y~;DcyBxahE*<wKPqa_ah7(!s zcdNShHdtLDSSUC5whg@6f6Jd0>PlD`+}A(u^rn<&8zczVDn%^A5dk-Vy_mr0qL*uM z+kH(G>dhnCDc>o`r?(AIs+^*rfe)ECTkV3CYD3Q#19fXQhe<>BD4P`WFJ{4fglrGp zMC#o(hLNz<j>R_6BG%EOWFS0kBYlhLR^aX`ly0}L;y&ATq<D7AgzWX2zq|M#3rJV+ zY~ru#z{ph6{`?|M#ksdVD)O<J(z0c>9Kgir+g(JSTR7eC^Kd70rtk@Qwh@u3M8?jc zvgkQ+ER2q@6iY?Es?2yUOPXy52HHmmw09OlCy8i1JSX$cFQ?Kz?WxLaD*;xXXdOZ= zBkjariS2=U=4{ztOD4WdLby%7@-N=%81G7r_onmAC}*~wh&dH`ElcXAaT1YCg!*3c zydPyIQxoLY1}B)t!AYV-sVm|=v@yqXQI~?W4Le?d1`+uZEGOQ|ee*VG<y~iZYo4yP zwK;cBbfr)o@_cgCkvaN83~AMRDnu5VjmvTry#Q0-*vnxL6@R&K_CRbn%zkeHvmO>f zrT|&74wW?}lFB{`<H*l>V02N9RseY6=RHwR+vczuOFPU6KW$IutXl`cwNkIGa12qG zrJ%bP3TNk7J?}yS3x6XEWxoN1EKl;n-Jr)OR82@8A-lLcqJ0m!DhivFnJu)<b#F6e z@*S}uGvu;>P!CIZozRj3Dupfu>UuxP6njtRWN0x(t)#GPjJ(W*QX;@KZebajIc;dm zCW~hL0jRsrD=aVq-P|3Oy{?-lW2lzd!ihrjVFr)oLbOS5oQOiE*S-!;?Lbx&bB@wB zIBCNkoH#5Y8I#5PlHx>EpLUEIfBnTV;pU<C7{c6XR=Oc|CcXpIvYcYfGS|3A|8m_r z{YixAwt8j1E`S!oHy!_OF<)mOPQYy%do`Tz?OthmA7GhuYZ2YqElQpW9u=sghW(BQ z)0wD{sar<Ct=s5cH+lIzk(2SI7G7*2(vZ9cksAReRW!l;Db!{3H9DXYw@vKORQjWs z(1L0nGxct#@5$_4Zx51ObuxCO1-7$k>3R%n<XL~r1pbnz63_7UX4}c+1!uDe1>fkZ z!YFhE-!>M@7lKEDX})s?nHWmd;*DDNM6GEm7PaY{ePtQ7vU*E6^Yo7t_xmKXg?pIw zLetbL($kGYR?TwDFJ{6?y@??DP->A;k*WI-u5h`r_Fj=a1?c8CaYv_fx+w3Y&sz)# z5l!Eerg8T>?FtY$ym)%@xf}a@V)bx@rCghzp-=;#(K|s@NOO*IZA)NzB23n8Oyp`N z6Y_)!pjq5GpOl;|9mspLVAjuk4Swf>dB>Z+oWGfksTiJHt6LL8{)`TN&}5mlo&S@f zn?k$j;4E88b8ms}U06xznINvR%znonws$*X0nXu~KR;D<HSVfq&JJwBWW~a;>&0=; zq1MxLBj~1VFmZ3_rpJ&0B|edG0LL4z$TA%JtOE-~I<eAvFsPO!FmC>HfCXompV+wy z8-&6rt-RaR;6BG2HZ5IoYkQ!W1K80!*5H1C5|T&@US7!VmLWU9nG%2IR0sf%g(q;p zir%R2#OCiM-FRbfu?u|_l)-Q7I{}F_K#B)nXF9wXSLm-9xO`&}clEL58GaMK6`1Uo zQKob~3zs=o{h-kD;27bhfCkdw{8=X?mD$rB(iIfJLV2z}Inma$btemM>{3VY_dH`c zRmH*W_;0{4Bi*0y!=kq3gCg}!KzsqQv(?<&2%Y|52_E_JZZE7axCF6;pWKz-h9;(1 zFEg|lB<E_(<>Dp{TkLtU9p<PabxBfte!h)MzH~=2$ruV~Q|!b0BP0Wv584wNc=W<P zF)PqC*q@tBNV$wj39Wk;H>c8X{8!)$h;lT}wYiX`cFvH{sCC$IJ1nrkGsX1R-c54t zLc9jBHVaK(PZqQAK)*w|rQxaCi@4yDsR;BKp_0+QMY4^V@oQdty=y?g5jigp7$EqZ zjDUR~x@7qfAlguTFi<0JZx{E(?05$3ZrE!(`+7JwC(6-O)0zPfL-;<ERZ_*6*uEqt zUd4B`&U+BepDs7jWeIx@Fs1StJ)4$Gq_bPR;XeDWlp(TqW7_CGp@csG3j`(_75|g| z#3lVBRcK`j?0qf1(S%3nBenyVCW2devBLNzc;uHlkTo3aZMq8a$EpTTymT;8km4In z^^`XDCQN?}yl%xiQFxvE2i4P_A@d&UZM*XENllCrSODTgR}M;vAjIK3t}4WPf%X^v z<@RIaQ14&z^clex8~HfEH1cY-d4TH}ZLUBras;kqvJ9zZ&8+jhyneYL$HF`$3o@Jx zwh&3_*O+5!s6=|WOnBPJC+!!{{8(!(8H@EM52Be?>9#k~GMZLtGy?nM#)>2+T`kNj ze-Cd%!Vd{3rx0cOIo+1L-plN7F!@)*0?vWum?{xsvwILKF<=UycOWzqNrt^1DAHo{ z&>l4+Ab^}}aY{#leq4;cq6#<-V$Ho7UKVZ81@Wh+CFOY)SxBEZUOMd5^n&4mJBI5y zhiL&%RP$EK=dU%dsx>v_%dKWSAnH{~OU>To6_twC8<j8ls8_^nkj#rIL>@+RTFwOV zjN#5sZh{G`WWFrn$+vV8xa_EdxGegTh$iG5fdf8|IkR2eF_u{^F!2%tv7EYty{ytY zfTzxF4)ngPoP_WTG|Fer08u&Q$%>o}_7yWw_VUke{^I-nDIPLL`#{~ep5)0hW*8ez z$=vvIc7ys0bTt^Z4cC$pSAr8jP+)*}S0n5;J4~41b{%cIM*fv_$1_a{7~CzEGF*%a zmo!~DyV(mH=a!<A!fPu}tmX2({yhO1q$y&V>>N6aTXY|l>8fd_G+w#(nF|q5jcLBA z13?#dl>PPCA}RNzqD6oVO(@OKym{I-Pa5JmLRwqW$<zg5o80shZ~1AVdls4YzSXVl zr(!<o*^@`vpQY|mKhk}so>FBiUBnL+P2)@~J(ec|s_sm!R2@$OKicGYN*2GqU(J&T z{Lqn)*=vxuAX1Gv0Dk!C`pCTtlDrGq_gKcHI?^jian>rS^UL?G0{-ilaNK#DTyw56 z{Mo5FbQ?H<GW^UdPMfKcigT4RUQyr97#cg3-rXvrw>ew~5Kllovle5o!-n7?EA%~9 z%jQnBip8H@%a9KGo;gZW59-6s%P>_Y62@fk&z9tt_3vec<8wZNl}y-DPVJOG|Iin_ z626Fx(_8z21@R?Y6h3=m$wyZ(m0~u^gGm$C_>_E9bIWd}w}}Fi6`vO0&SEgSdVWB! z70oGSTwI5)%Dq)n3w0Upp_=|g;_;3OZw=}>WJUsdX*M=A4EsAwYD>0ZPrKc^Y`%(P zR4QJgyJNu<OnaNaCF*q=HK76e!SY}z`hK|B%_pdp8=PlOq@!!2%n9!%X&#=+f9Bzb z%r4-tM4JWqGNz)Y$bXdU5iQF(Sd;XLYeiU~AEU0r0VDHpx7$;)E>4aNup&3279U6_ zdbsfLmw#jb+-(ai0SJf=$M4ESh<?4mDs5->--^XS307Zgwt`pJ8{}aNm%u@LRcdGx zw~H)F7#N<BKf6S=qCc}~=CQ`QQoIUa5cP#_7mPC8fbx6UF8)QITPOnA+T?jls?QGh z2}c6c`IqZ?JCM2RXZFtSfYwvhDsiWn*0Q$BbgtU~+gsaZS7YNl-IdGo9#4M6!C3bi z^6pj*QYX5|WqG~TI9YKx&*h<(+N^g;P?JN;DjI7GIrc**J<;8Q>IpX{7#kW5V(1H5 zz5AdL#5;!Xs~elu2h{fX{pR6_V=3+&^ruJ{iTx$`s^O_)RYD@?{ol+}(o43PDCFcy z>6@z&ig(9lnQ&Je#^YG*qG0nV5izc-nDi1Oya!vptC5L&xq!LbWas62!Jk9@Hgg$u zcf|NzytpAfC_?Eo)ZG&ywyD+)KyrtAk@F|5=o#Mda4t2W8yW1la)U@5zE9jn2t8L( zX81%5B2%>F4iIQQ*!=|^;t?PSN?@8gFwrSJ@S<q1`&eFRWLb;p?|IYWvEU?H@R5%f z7&gpUo{NBPGHzv)nB|tYvO%bRtW8h1L3^QxSTD{|thln?XU6)GQ|?~J${<z=@_cC) zfj0>3$#y8xt&xUbuD-u=7}9#eLWR72-qTT@xu+BTcA6}iClYMq3D|3PS&w~_olnHK zbbUG}X3XIIUV2VpcbYSqR^lWK`E;G4pb|N_JYdhO<a8KzpFM|6P3?^$e>-P9g;3Pq zx#XGZHE!5Xc?m~}&3$AbIXJZLI=xQV><&VT5CXbQ&*Kz10ue(bo$2A6<!-#F#^4ee z^r+g26{<7ixbv3;x4upZtp>1QOcN*>`p;EOKRNXLPtn*{8w3F-Cleb(>;Dq;Q;C(4 zd?J7xq=(1C&}V+H(IjuWE!QWIPhSF^7YZk!fUfOIo+QzqwU^5k7P>3Y8U%-;?GA!O zHYcntF5ohIP^By2<KyQIw$K;QF?@b*0@!6SaTc=}UcAg}P`f5cFqRz7Vl2&mjag$W zk>K2uO|W-gA~czK@O*61M(U{K*rX<VU_`A6YI06iD0t-Ksa@Ohd>X`j+=FR!L5*bC z8%ZNoC}V;XL!Kpb>sP)JkSj_sf;rwMx2$<+g%bK77T7~8tSw-VD@GV=JA)2g5Hs@& zN(X^2sMAj;J;5<g?d$Y-E2U5s>fpbBvQ$s%Wr@mKo`t|+60qbQv%_fRc(1N8*2fDS zc~Y)?i3pyo`Y`?2GK=TmHMB1Sk?@)-KhzR}Oj=qWo(Ut-uUx}_lC%xNatZzBfmEBJ zSB2ILfPtS-VxP5RivoeD?|F1}MKFC}S2DXwe+>&i*)@^(pNc<0Ylm@t;ENoizkQkG z#jnpbKyf#qNVc<uiPKlfba4Y)<Nw}?Kz%ngJQef@M0s*qK`F{a#ooRgY{3@|b-1}$ zS_J_J@-Uc6bnc8C5|HFfo7y=v_2s+((vMI+hgR8wR&%vTZ8yhur)&VjC9}q7+S+OL z<V<FIZ9W{UEA5Y8hO0nc7zM5mc85rPurm^=tbluqo*rqjt<G4a!J@_1Sgd7QW|{;( zNMHbc46CTxls;7^sO9!04KJO>sT*VPwT{GWW9AfDFmg(z^eN2;&JR3~wRYIg?8~`b z6w+Q}ETeZ#j>1Z?z5425VK$AnXI=J;)o?YW1AC@*n=7rc0xy8rmLo~Jcb!bgn3ceG zv1@S2g~rpP*}ia;hD~CRV%Kn2XA_Ux$o_4-22CZ*sM5r!eGy6Peeyw==5WHgAUBr! zfvRYibkq^Pj~pB0`BIi)Xx#xu3H)+%OM<GI_n|L1%0&tE=Cf}8VYiQr@Zr++O|YIg zf_bK1TCWD|Iepp1!F|~CH<)j6Zh>`sS+HY@3+2tFUh{#~*CgyA#2A6><Mazz>lqfn z6S5O{6{Wk3D3`MS+HG^VfwulGBaN;h`#huNIg<4%zjQE;0edb^GBt_26eM9Eg~2<= z%x&8wNd;sz2J(b`T`Vn+b%GZu!pg_&@u<wb6HfevPlYO`c-HO59kIs<Pcs*j&^ko4 zVK)h+=`RRosT;d;khQRSp3ntcPoVX+3@Fx<!<6)TmDr#Yk4Q*Mw3H|Mn<+qr=7!j~ z6&QOFuqjgjUJ@T3J5dpIRXjOdWKcULp8W9DeS^ALjV!At#!!-cxlfGBlz#X$`~Yit z869%NssGa{!?E}Q{^iG<()f;^$R=paY6=XuJ?%I)dR#uu;~D~WL0#QpMh;XM?Am_z z2e}e2^*CWb<a(mr6#2w3Cmk*c_Dl17<j0t?ZXfR#JL_<`i#~=g#<(Y5(TYiSs(FP! z-FKX=VI^w~sKOb0hc4B&x5qO1Nqq?~r9pS)%q9Ps8<b4fK;)*ntn*u{k&+AqMX{qV z9FfTuuPTG&A5ysBP0Py!`HQvUyoR2Q%=leA@?$FBbHc+YB>44I_b|jc_M^Ast*GX% z<B{AkAL-#%w~A3`H8<1P>~cER`C{E`DzN*%y4r>@ti4A$Le2~6EEK|BE&%nFopIQQ zN!-D9pX<=ija}?3M}Wur)SnR4!<y&Ps?ooQ5Q%YY9T+)HXP;`odjodD#NGGIN85;k zX2#*IU2}9(|0ti*#WG%{Q5*dSmHy{^ERW*wE>Q^=N{TZI>K-5OX+PuZ@ecEdP)O|3 z;Z49IgbEtgSJg(<bB=Jn`E0{ZM>*(Aa^$Aoi=5ZV6^_E4HzP)mn?bbRzqSk-Q@}P! zU^@l7uS{R0FQ1#*uh%#!jP+VDBI7|deK+xz-o;cMwsFQa_N6oU`m|HL^uTLD=QXI? zqFiDND9*>fT!W9Zuh{5;R})jH-(6Au;dQ~kD`bIM)20??E{+DjC_(m7K9a=~L+3%m zmtNX7LSUw(wb78YdD4gQYKDwb0w6B<B)1k$1zC}KVe>K=Xyc%RR<CmY)F=4Cyihag zHXmsMD2`F$*2<+nQqWL$J=V0d)a0~t{BV9%)egGUN!U$iw~`pnr}4bik*c+<-c-dB zXm4e~su?{jRJ^y1H(K^|39E9=KSw_k>PAvWSvJs>w0h2R385!%w)PxhWr&M01bMie zx>a1ez2u_4;Q$qR#^a%(z`bD;W}PcbW;gZp$;XJ(jj16;20aY3xp5(V_)^EWM`}Gr zK#ADYB0DVWY&9JP_oH)FDL~K(Y0HNT%jo5+7MAC6`q*B*<L;e6P1prK4>BqP)IfOA zSs1}p4ht#5?g87B?XYTl`HxLvWh($kg4e|Fz2Zvohr;hXR?n)(=s&<E9e8_O+BPlg zi$Cnv<w;LPG?LbB9u)cAW=<oIx(;->V%ugp%$J_YTVFooJk<#&j9b704}aM+b!QM* zY2B{6NUDF@2GpzM?B-{6Ghg#rk|qw*Qr=FO%CA^HN`cxwni?*?^I8;o%^2I|#b!@H z!~kFZVrVLm*xR}zG$0!nJB)j{!+gufR3E<Rf|o6}ukoT5lAOePrNelOmN62NWT|<^ zg2}6CwswMeDfXTYyYFcDzC;i0Lq_HGrhhbd>ieNl0$mvb9e%%PXc-huMH^XTw*p?1 zYyBDhW(uaF%N2hMy<Xc(8y{YD$l&~B-lvMEh>CTWakzvUi@hY_+R8p{u`b*vcrP^U z_*g|+yWK|d2olI`sQ^ThBwo*25*7;P@yH3tB(f9HU$-isz0RnuWHIEzUyNIb?n@Re zv$Du(b|ul3b3Fq0U>?6%DxBrqHZ@M!(Q9Sr<$XXSD&RZR=lmi8#Wa<kw33mEbMQ1_ z_bJ;%XTadY(bGq+t??dIScgjH%1I*Z487g8hHyF~vgmZ<ZQc2u$Bqj-pmeXQ%b5jP zdwJ40#Z8LmO=LXQ=h-B70xN;ud3(78-}QNRZ#_CbzWaGi58c7}`d&WeUcM=;M|qRg zi6OHjZzZ~Hl{4B3Z`r|jmuDPges6fRL^F@V+E{SN{K5&MxQn+|*HUG(BDOBNCeCpo zAO6w|*u5rwkO1mdYX0_MrTFz7#NP+2h`WiA%P+KHRd4+8dNCfO$-IJU*=r0FS}^jE zwUTKZ4ddn0*g>VOpR03FJ!gJX7}xq)vi!L65L~h`COI7w7PQN!xMG^TmKZsOTAK%u z#7EYSymBa>Y&`4@Ffm&lxog|JGhG>B<MVb$?C|vNx4&Pg={)8V?ACNl+)>Px$u;Ig zhanra)@5TBV{@8(le)od=MZScTHK2=8ci<d?l|+p_-LZ2&UAqrb;vAFgO+GE2w`mA zR=wCIp>kHIuNW>^0PQLiQ-@U95r?P0sc?spnX8XB-Fwp8ZN9nk*gQNY==j2)0kCP> zDS3wH9LV%ani_3bU2|xy#zAU$rwL<`uAe~6y>{(&G8kQVUiZh>m`rur<PBB;dO1$m zv;pN6Mv3Xkrcry1>~bZ0XVL~QQ(q<_ClM)5o8+`+95hA?X0lOj&2f6?i%}xEm~y3R zZA1w3h^*;MJ*GFdRrP9o(a}EeSy$0MRB1H>ND#EI?o(ILX|D1yXsML7Jz;PiQelZ+ zp!i9t0BZQ}Y0c!zH|4A21GdDR7i)Cpg{XY}^=@lm1vWb9>y^p4F^Fj{5|XH~U(`1y zf0U&kUb4c0uQ(#`!MNRwE;%*DP}`saRhM}Q@8)WSInEKkDq_N)ih@A^4cDIuzpTR1 zg1^TRqQx;vVRq~}7XnA(a3&`_p-X}Rp+M!R82&a9yRuU2)qbcH!*(OuBG-ZxL$7^3 zk&b$I^~5I@OdQRRR`nvwa|Z8Ax*#R#RSH|9#$u7?>1oDhG*RHFDlwSr4bi&61QLwz zDLzl|vh{cbR+{+2Riced&uLkYy9`dK_ScE8u`N&ueqg2cUruA%=)P)#35CF58vwV> zIFPBlmMmvWShXzwjAC;X9Q9dnE`&F@@U8Utn=<U!D>nx1ySEfLX(0((;LiiMhO*{o z332vyIVs;A+_1A?y(oW|?Fl2oUa(^_iON_+oYqiYgd}-iq2eyFl8e*2C7b|Q$7#)w zm1s2=sH^Fdv2u>d+BWU{?4KqFr-5CP>KbEH1xpYDVVij6M-c8AG=ym^@?d!I(P`9u z(W@77VDq{wy0<#R`)C@Tr;x*YPD61$^u=U&KnFrtLk+}c7XYQ}!}&%5t49-o8#I6j z8$BWc@|_PmISg)MZFq}`=(Tu&Y0*gn=!zUT%<iO(bJ&q&@!eooV0=zzngs@ul#pMd z$GEPK@JFx2E8Cz30<m~*y2bs?L<JLr>R6}HnzGC1I3zr#o#GHqMQG@>OzQj7okNAF z(psjhjkl6sE-6TI^GhnVg0K&Qnd~;28l$D{!$=pSZL9m)_hz5f__8{k;McQxsl7yL zoV4+ZL@DetHhsB+u&|Sr*#=j%+t!eitu!F$RMK>tLL_&GeKR_!oe^eQ=FnS3U9fs4 zI?FrCX<AfQI?{bxqu&j=l&C12A_^Hz(f8tb$t^{nY<x+JA1%mVvzw_@s^=OH@uSyP zsoI-8K`$1jc{E<&)Q9Y6&hH4^Y-U2m-xgb@f;{4u+qcLp4)CnBeIJ(L_+Ab!yaCoV zqHa~ks%0t<;(E@7HwZ}>l<!$Bb@_YeC#F3R*C>H>RT``+eW}G!(+Yec7JR&Y?WJi( zmoa%r*|6?kWI2MyMWFR&UR94W?=gsTJxJ}_*g_YkdUWL!owBrj-lX=Hx;)8+BIbFr zftcCqOWQ7{96mH7cGBrD==xgg7+$j^gyKT_a)O9QZ?{T>TX!jrkd>J#Cm|;2;tO2| z=43{SY5NJhTQKQ*&oeNy$u#WO!de&b$r+usOzH|f+vA&o_9PCcYXVad((7s>b=O!Z zxvTY)LL%1i&SDV@+C7(o`!I)3_ln}{m?q?=Y~@fKh>zj!lY5>N_O3$Ml2U5KPx+(7 zN0LYrf4JaN?NRvbXSVht{+PCc8`(XyfG??_f2D8e;jKH>`WI|T!;Wbjq<ap|pdHc# zcZXYa+}-7R%xU)kU99Qjfm1%oCIkd532XU@l8KfuO7{)0WK^?UMy_3ATW)5cXf9gV z!<e``YL#fXdrN3Su{k$~j3=D97sH--KCM2M#07Mg9CW&0F{Ni;5<DS4TJIBU`0T#g z4q=o>P9zrm*ZR7KW`bM%aMZ4>;lijsSslVlc+pT}&WfxFuQSMv0}uM1%mqJA$7GWa z6pIIode$f6LrBHlm1tMmunGE`=P4W`HIGYvT#t8kYINF0AA{{c=jGrCMA7YO`<&7m znPRW=3T+R(iyAEZD5LA<lsN2*9IY$p(U9pGP4&CPYL9q4i&k0rNeSs^tRCa}7O8F| zj)YNhVJiKT1Exw-a=k7%VI!sm2wBMy#uEq0=Qwd&5U9gARe9p6=|)eu`M|ywSGWXi z^qxS1fD9viz4ZkJLkB?zfrbVev49YRRKLq7+yj2!2CSk%23}8p3>gt+0a^)JQ95Y} zArV<65fx<!5a2`of2Xk=AJ$9)e%}QAMgJiUJt7Fm|E7@<kd+V>QBr;(Bl?f2HlYs0 ziGdJ%;O|#epl^W;RG_kRG@~>7OHhi=$l8MLJ1b@ZM>7{2pdviba?Qm47dPlXx4gn5 zAS((u#k2^#&-gl#^es}6f5-WyC+g41pS(8g(F7)M06uYiwe0*BfoQ)={+9!*<1+zM zpe4zFKtG#={Y<hJTboh@HX#9ctHk{SNeuH3NP;G&c8)-II!7mGB^SfLyYr_?nxB#U zo_+1QKpZD<fAIW(gvb2@q?9eti{n3y`%iO$7A(#t|H8B+u8@)fc4!K)Lw{`M3h^H> zWh|VWfPQ@cp#n$BpCHi$Fq3A1NJ*f0`j5@bc=iX#zgcbujwXNJ%$8|Sv|Ql8_W^R* zf9Tq6-~sy2ga7Yw^MCDC&}KYbVj#*CIDmc}rk9j|j8g*IG1;2^%l?~tkP<BiHf#(? zWBj3EFKPaO^yfWjy7%735eVrA4ivs0K#4E@0Hk7T_5U+Gzju`X@nqu7@bA=r>AUa! zoPSK-#rj{#|LUpVSk(V~Fn@15{M8crTjX;6d-DGbxPRIH@BK7?9A#=eKOijruWrUa zH|Ben#;-;|-(p?xH>CfwTj$T*@7>LQyk=br|G@pFquD<@LjKJ8-uCLNSK7B=k^Fbg zA3CS~4E^4B>8qpGw|FJ}1N48^U;fBn>u1XM)-XTrI(OM$QvTNt=KtpC^fUK+i;S<v zGv9(s@%P++wW0YL^}UMbSI>=aQLge^)V~~G-zzMBoxuDS=O(|*`v;1gKX3c@GJ`*k za60qfF#ev4`Df+EpE=)Gb$=Bt{1(v`f5!Qj&icO6_{Yu)@%|;?4@$*<Zp8PC?O$t_ zzlE~z?->8G>EADkeqE{m7WL`BO#91q`=2-V`_;N1uP(+}zs&l(<<*~)e?RN~b;0jj z5a;|l`5!F*{S5hjw(!SY+EDOI$ls&#chmVlGroU@`a19UEsRQj$M}a?NO>s;-~$;5 R2np~f1o-$>Q}y+){|A@R9n$~+ literal 0 HcmV?d00001 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..b52fb7e --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,9 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.5.0-bin.zip +networkTimeout=10000 +retries=0 +retryBackOffMs=500 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100644 index 0000000..b9bb139 --- /dev/null +++ b/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..24c62d5 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,82 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables, and ensure extensions are enabled +setlocal EnableExtensions + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +"%COMSPEC%" /c exit 1 + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +"%COMSPEC%" /c exit 1 + +:execute +@rem Setup the command line + + + +@rem Execute Gradle +@rem endlocal doesn't take effect until after the line is parsed and variables are expanded +@rem which allows us to clear the local environment before executing the java command +endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel + +:exitWithErrorLevel +@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts +"%COMSPEC%" /c exit %ERRORLEVEL% diff --git a/post-commit-scripts/._commit-metric.sh b/post-commit-scripts/._commit-metric.sh new file mode 100644 index 0000000000000000000000000000000000000000..f79ec62787c9ce61d07591a146666a775513ad85 GIT binary patch literal 311 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDJkFfg(LX&|4`97q!Z9795aAj-fxwgB19 zXxc!ggTy@;82FR(bM+Dn3UX5QaubttAPWBgDQgA>QG{G!X<|`gUP)$NDg(p8JL@zo zCd}|!GtXOQj?J!{RS-?2l7;#P1{OxvW|m2bW+?_K*1<W6$=TLU=7xqA7H$T*juwWF bx+ca3K$d~4v#zC!le3Ylg^8JkktG8FMIbF1 literal 0 HcmV?d00001 diff --git a/post-commit-scripts/._runner.sh b/post-commit-scripts/._runner.sh new file mode 100644 index 0000000000000000000000000000000000000000..df18094e3e4df62cae8cc2d7376b4718cd15decf GIT binary patch literal 212 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDI}@gg7w@vi_e5x_AdBnYYuq+<pU+n{M< zM2L$f=jZAr78K;9>J^qI7A5ADWagzZ6zUroSQuKHStcc#S(qkSJ7*N-=cZblyO_DS lShzasTDrKH>6$pZnd(|P8yV?3nHsnlxVXAn7+N?o0055kAH@Iw literal 0 HcmV?d00001 diff --git a/post-commit-scripts/commit-metric.sh b/post-commit-scripts/commit-metric.sh new file mode 100644 index 0000000..0689961 --- /dev/null +++ b/post-commit-scripts/commit-metric.sh @@ -0,0 +1,1551 @@ +#!/usr/bin/env bash +# commit-metric.sh — Cursor AI Commit Metric Collector (Bash port of commit-metric.go) +# +# Architecture: Two-phase execution triggered by post-commit git hook. +# +# Phase 1 ("start") — Runs synchronously in the hook (fast): +# 1. Checks if this is a normal commit (skips rebase/merge). +# 2. Gets the latest commit hash from git. +# 3. Writes commit info to a temp file. +# 4. Spawns itself as "continue" in a detached background process. +# +# Phase 2 ("continue") — Runs in background: +# 1. Polls Cursor DB until commit hash appears (500ms interval, 3 min max). +# 2. On match: builds request payload, sends to API. +# 3. On timeout: calls /error endpoint with commit hash. +# 4. Uploads dangling prompt metrics for repos matching the commit. +# 5. Retries failed requests from previous runs. +# +# Dependencies: bash 4+, sqlite3, jq, curl, git + +set -euo pipefail + +# ============================================================ +# Dependency check +# ============================================================ + +check_dependencies() { + mkdir -p ~/bin + + # Detect CPU architecture (Intel vs Apple Silicon) + ARCH=$(uname -m) + if [[ "$ARCH" == "arm64" ]]; then + JQ_URL="https://github.com/stedolan/jq/releases/latest/download/jq-macos-arm64" + elif [[ "$ARCH" == "x86_64" ]]; then + JQ_URL="https://github.com/stedolan/jq/releases/latest/download/jq-osx-amd64" + else + log_warn "Unsupported architecture: $ARCH" + return 1 + fi + + # Download jq if not installed + if ! command -v jq >/dev/null 2>&1; then + curl -fsSL -o ~/bin/jq "$JQ_URL" + chmod +x ~/bin/jq + fi + + # Add to PATH + export PATH="$HOME/bin:$PATH" + + # These should always be present on macOS/Linux + local missing=() + for cmd in sqlite3 curl git; do + if ! command -v "$cmd" >/dev/null 2>&1; then + missing+=("$cmd") + fi + done + if [ ${#missing[@]} -gt 0 ]; then + log_warn "Missing required dependencies: ${missing[*]}" + return 1 + fi +} +# check_dependencies is called from main after parsing the command + +# ============================================================ +# Configuration +# ============================================================ + +# API endpoint to send commit metrics +API_ENDPOINT="https://cursor-server.meeshogcp.in/api/v1/add-commit-metrics" + +# Error API endpoint (called on polling timeout) +ERROR_API_ENDPOINT="https://cursor-server.meeshogcp.in/api/v1/error" + +# Set to true to skip API call and only save locally (for testing) +DRY_RUN=false + +# Database configuration +DB_RELATIVE_PATH="Library/Application Support/Cursor/User/globalStorage/state.vscdb" +TABLE_NAME="ItemTable" +KEY_NAME="aiCodeTracking.recentCommit" + +# SQLite configuration +BUSY_TIMEOUT_MS=3000 +MAX_RETRIES=3 +INITIAL_RETRY_DELAY_MS=500 +MAX_RETRY_DELAY_MS=2000 + +# API retry configuration +API_MAX_ATTEMPTS=3 +API_INITIAL_RETRY_DELAY_MS=1000 +API_MAX_RETRY_DELAY_MS=5000 + +# DB polling configuration (post-commit: wait for Cursor to update DB) +COMMIT_POLL_INTERVAL_MS=10000 +COMMIT_MAX_WAIT_S=120 + +# Storage paths (relative to $HOME) +FAILED_COMMITS_FILE=".cursor-metrics/commit-metric/failed.json" +METRICS_OUTPUT_DIR=".cursor-metrics/commit-metric/data" +TEMP_DIR=".cursor-metrics/commit-metric/tmp" +LOG_DIR_RELATIVE=".cursor-metrics/commit-metric/logs" +REBASE_MAP_FILE=".cursor-metrics/commit-metric/rebase-map.json" + +# ---- Dangling prompt metrics configuration ---- +PROMPT_API_ENDPOINT="https://cursor-server.meeshogcp.in/api/v1/add-prompt-metrics" +PROMPT_DB_TABLE="cursorDiskKV" +PROMPT_PERSISTENT_STORAGE_DIR=".cursor-metrics/prompt-metric/composer-partialDiffFates" +PROMPT_FAILED_REQUESTS_FILE=".cursor-metrics/prompt-metric/failed.json" + +# ============================================================ +# Logging +# ============================================================ + +LOG_FILE="" + +setup_logging() { + local log_dir="${HOME}/${LOG_DIR_RELATIVE}" + mkdir -p "$log_dir" 2>/dev/null || true + LOG_FILE="${log_dir}/commit-metric.log" +} + +# logWarn writes a timestamped warning to the log file with [commit-metric] prefix. +# Falls back to stderr if the log file is not available. +log_warn() { + local fmt_str="$1"; shift + local msg + # shellcheck disable=SC2059 + msg=$(printf "$fmt_str" "$@") + local line + line="[commit-metric] $(date -u +"%Y-%m-%dT%H:%M:%S%z") ${msg}" + if [ -n "$LOG_FILE" ]; then + echo "$line" >> "$LOG_FILE" 2>/dev/null || echo "$line" >&2 + else + echo "$line" >&2 + fi +} + +# ============================================================ +# Utility functions +# ============================================================ + +# min_val returns the smaller of two integers +min_val() { + local a=$1 b=$2 + if [ "$a" -lt "$b" ]; then echo "$a"; else echo "$b"; fi +} + +# sleep_ms sleeps for N milliseconds +sleep_ms() { + local ms=$1 + local secs + secs=$(awk "BEGIN { printf \"%.3f\", $ms / 1000 }") + sleep "$secs" +} + +# get_db_path returns the full path to the Cursor state database +get_db_path() { + echo "${HOME}/${DB_RELATIVE_PATH}" +} + +# ============================================================ +# Value decoding +# ============================================================ + +# is_hex_string checks if a string is hex-encoded (even length, only hex chars) +is_hex_string() { + local s="$1" + local len=${#s} + if [ "$len" -eq 0 ] || [ $(( len % 2 )) -ne 0 ]; then + return 1 + fi + # Check all characters are hex + if [[ "$s" =~ ^[0-9a-fA-F]+$ ]]; then + return 0 + fi + return 1 +} + +# hex_decode reads hex from stdin and outputs raw bytes. +# Uses xxd, perl, or python3 (whichever is available). +hex_decode() { + if command -v xxd >/dev/null 2>&1; then + xxd -r -p + elif command -v perl >/dev/null 2>&1; then + perl -pe 's/(..)/chr(hex($1))/ge' + elif command -v python3 >/dev/null 2>&1; then + python3 -c "import sys,binascii; sys.stdout.buffer.write(binascii.unhexlify(sys.stdin.read().strip()))" + else + return 1 + fi +} + +# decode_value decodes a raw DB value (could be JSON or hex-encoded) to JSON bytes. +# Mirrors Go's decodeValue function. +decode_value() { + local raw="$1" + if [ -z "$raw" ]; then + return 1 + fi + # Check if valid JSON + if echo "$raw" | jq empty 2>/dev/null; then + echo "$raw" + return 0 + fi + # Check if hex-encoded + if is_hex_string "$raw"; then + local decoded + if decoded=$(echo "$raw" | hex_decode 2>/dev/null) && [ -n "$decoded" ]; then + # Verify it's valid JSON (UTF-8 check implicit) + if echo "$decoded" | jq empty 2>/dev/null; then + echo "$decoded" + return 0 + fi + fi + fi + return 1 +} + +# ============================================================ +# DB helpers +# ============================================================ + +# read_db_value reads a single value from the DB by exact key. +# Uses typeof() to handle BLOB values safely (returns hex for BLOBs). +# PRAGMAs output is suppressed via .output /dev/null so it doesn't mix with query results. +read_db_value() { + local db_path="$1" + local table="$2" + local key="$3" + + sqlite3 -readonly "$db_path" 2>/dev/null <<SQL +.output /dev/null +PRAGMA busy_timeout=${BUSY_TIMEOUT_MS}; +.output +SELECT CASE typeof(value) WHEN 'blob' THEN hex(value) ELSE value END FROM ${table} WHERE key = '${key}'; +SQL +} + +# read_db_value_with_retry wraps read_db_value with exponential-backoff retries. +read_db_value_with_retry() { + local db_path="$1" + local table="$2" + local key="$3" + local retry_delay=$INITIAL_RETRY_DELAY_MS + + for attempt in $(seq 1 $MAX_RETRIES); do + local raw + raw=$(read_db_value "$db_path" "$table" "$key") && { + echo "$raw" + return 0 + } + + if [ "$attempt" -lt "$MAX_RETRIES" ]; then + log_warn "DB read attempt %d/%d failed for key '%s', retrying in %dms..." \ + "$attempt" "$MAX_RETRIES" "$key" "$retry_delay" + sleep_ms "$retry_delay" + retry_delay=$(min_val $(( retry_delay * 2 )) $MAX_RETRY_DELAY_MS) + fi + done + return 1 +} + +# read_commit_data reads and parses the commit metric from the DB with retries. +# Returns empty/fails when no commit data exists. +read_commit_data() { + local db_path="$1" + local raw + raw=$(read_db_value_with_retry "$db_path" "$TABLE_NAME" "$KEY_NAME") + if [ -z "$raw" ]; then + return 1 + fi + decode_value "$raw" +} + +# read_repo_tracker_paths reads the repositoryTracker.paths key from the Cursor DB with retries. +read_repo_tracker_paths() { + local db_path="$1" + local raw + raw=$(read_db_value_with_retry "$db_path" "$TABLE_NAME" "repositoryTracker.paths") + if [ -z "$raw" ]; then + echo "{}" + return 0 + fi + local decoded + decoded=$(decode_value "$raw") || { echo "{}"; return 0; } + echo "$decoded" +} + +# ============================================================ +# Temp file helpers +# ============================================================ + +# write_temp_file writes commit info to a temp JSON file for the background process. +# Prints the file path. +write_temp_file() { + local commit_data="$1" + local dir="${HOME}/${TEMP_DIR}" + mkdir -p "$dir" + + local commit_hash + commit_hash=$(echo "$commit_data" | jq -r '.commitHash // "unknown"') + local timestamp_ns + timestamp_ns=$(date +%s%N 2>/dev/null || echo "$(date +%s)000000000") + local file_name="${commit_hash}_${timestamp_ns}.json" + local file_path="${dir}/${file_name}" + + echo "$commit_data" > "$file_path" + echo "$file_path" +} + +# ============================================================ +# Git helpers +# ============================================================ + +# getGitEmail retrieves the user's email from git config +get_git_email() { + local email + email=$(git config --get user.email 2>/dev/null || true) + if [ -z "$email" ]; then + email=$(git config --global --get user.email 2>/dev/null || true) + fi + echo "$email" +} + +# epochMsToUTCString converts epoch milliseconds to UTC string with ms precision. +# Output format: "2006-01-02T15:04:05.000Z" +epoch_ms_to_utc_string() { + local epoch_ms="$1" + if [ -z "$epoch_ms" ] || [ "$epoch_ms" = "0" ] || [ "$epoch_ms" = "null" ]; then + date -u +"%Y-%m-%dT%H:%M:%S.000Z" + return + fi + + local seconds=$(( epoch_ms / 1000 )) + local millis=$(( epoch_ms % 1000 )) + local millis_padded + millis_padded=$(printf "%03d" "$millis") + + local formatted + # GNU date + if formatted=$(date -u -d "@${seconds}" +"%Y-%m-%dT%H:%M:%S" 2>/dev/null); then + echo "${formatted}.${millis_padded}Z" + # BSD/macOS date + elif formatted=$(date -u -r "${seconds}" +"%Y-%m-%dT%H:%M:%S" 2>/dev/null); then + echo "${formatted}.${millis_padded}Z" + else + date -u +"%Y-%m-%dT%H:%M:%S.000Z" + fi +} + +# toUTCString parses any time string (e.g., git's ISO 8601 with timezone) and +# converts it to UTC with ms precision. Returns empty string on parse failure. +to_utc_string() { + local ts="$1" + if [ -z "$ts" ]; then + echo "" + return + fi + + local parsed + # GNU date: handles "+05:30" colon timezone natively + if parsed=$(date -u -d "$ts" +"%Y-%m-%dT%H:%M:%S.000Z" 2>/dev/null); then + echo "$parsed" + return + fi + + # BSD/macOS date: %z expects "+0530" not "+05:30", so strip the colon + # from the timezone offset before parsing. + # "2026-02-15T01:26:12+05:30" -> "2026-02-15T01:26:12+0530" + local ts_nocolon="$ts" + if [[ "$ts" =~ ^(.+)([+-][0-9]{2}):([0-9]{2})$ ]]; then + ts_nocolon="${BASH_REMATCH[1]}${BASH_REMATCH[2]}${BASH_REMATCH[3]}" + fi + if parsed=$(date -u -jf "%Y-%m-%dT%H:%M:%S%z" "$ts_nocolon" +"%Y-%m-%dT%H:%M:%S.000Z" 2>/dev/null); then + echo "$parsed" + return + fi + + # Return as-is if unparseable + echo "$ts" +} + +# is_normal_commit returns 0 for normal commits, 1 for rebase/merge/cherry-pick. +# For rebase and cherry-pick, records the original→replayed hash mapping before skipping. +is_normal_commit() { + local git_dir + git_dir=$(git rev-parse --git-dir 2>/dev/null) || return 1 + + # Skip during rebase (interactive or non-interactive) + if [ -d "${git_dir}/rebase-merge" ] || [ -d "${git_dir}/rebase-apply" ]; then + local original_hash="" + if [ -d "${git_dir}/rebase-merge" ] && [ -f "${git_dir}/rebase-merge/done" ]; then + original_hash=$(tail -1 "${git_dir}/rebase-merge/done" 2>/dev/null | awk '{print $2}') + fi + if [ -z "$original_hash" ] && [ -f "${git_dir}/rebase-apply/original-commit" ]; then + original_hash=$(cat "${git_dir}/rebase-apply/original-commit" 2>/dev/null | tr -d '[:space:]') + fi + [ -n "$original_hash" ] && record_commit_hash_mapping "$original_hash" + return 1 + fi + + # Skip during cherry-pick (CHERRY_PICK_HEAD exists until post-commit cleanup) + if [ -f "${git_dir}/CHERRY_PICK_HEAD" ]; then + local original_hash + original_hash=$(cat "${git_dir}/CHERRY_PICK_HEAD" 2>/dev/null | tr -d '[:space:]') + [ -n "$original_hash" ] && record_commit_hash_mapping "$original_hash" + return 1 + fi + + # Skip merge commits (HEAD has more than 1 parent) + if git rev-parse HEAD^2 >/dev/null 2>&1; then + return 1 + fi + + return 0 +} + +# record_commit_hash_mapping saves replayed_hash→original_hash mapping. +# Used by rebase and cherry-pick to track which original commit was replayed. +# Stored at ~/<REBASE_MAP_FILE> as a JSON object keyed by replayed hash. +record_commit_hash_mapping() { + local original_hash="$1" + + # Resolve short hash to full hash + local full_hash + full_hash=$(git rev-parse "$original_hash" 2>/dev/null) || full_hash="$original_hash" + original_hash="$full_hash" + + local replayed_hash + replayed_hash=$(git rev-parse HEAD 2>/dev/null) || return 0 + + local repo_path + repo_path=$(git rev-parse --show-toplevel 2>/dev/null) || return 0 + local repo_name + repo_name=$(get_repo_name_from_path "$repo_path") + local git_dir + git_dir=$(git -C "$repo_path" rev-parse --git-dir 2>/dev/null) || return 0 + local branch_name="" + if [ -f "${git_dir}/rebase-merge/head-name" ]; then + branch_name=$(cat "${git_dir}/rebase-merge/head-name" 2>/dev/null | sed 's|^refs/heads/||') + elif [ -f "${git_dir}/rebase-apply/head-name" ]; then + branch_name=$(cat "${git_dir}/rebase-apply/head-name" 2>/dev/null | sed 's|^refs/heads/||') + fi + if [ -z "$branch_name" ]; then + branch_name=$(git -C "$repo_path" rev-parse --abbrev-ref HEAD 2>/dev/null || true) + fi + + local map_file="${HOME}/${REBASE_MAP_FILE}" + mkdir -p "$(dirname "$map_file")" 2>/dev/null || true + + local current_map="{}" + if [ -f "$map_file" ]; then + current_map=$(cat "$map_file" 2>/dev/null) || current_map="{}" + if ! echo "$current_map" | jq empty 2>/dev/null; then + current_map="{}" + fi + fi + + current_map=$(echo "$current_map" | jq \ + --arg replayed "$replayed_hash" \ + --arg orig "$original_hash" \ + --arg repo "$repo_name" \ + --arg branch "$branch_name" \ + '. + {($replayed): {original: $orig, repo: $repo, branch: $branch}}') + + echo "$current_map" | jq '.' > "$map_file" 2>/dev/null || true + + log_warn "commit mapping recorded: %s → %s (%s)" "$replayed_hash" "$original_hash" "$repo_name" +} + +# getRepoNameFromPath tries git remote origin URL first, falls back to basename. +get_repo_name_from_path() { + local root_path="$1" + + local url + url=$(git -C "$root_path" remote get-url origin 2>/dev/null || true) + if [ -n "$url" ]; then + local name + name=$(parse_repo_name_from_url "$url") + if [ -n "$name" ]; then + echo "$name" + return + fi + fi + + basename "$root_path" +} + +# parseRepoNameFromURL extracts "org/repo" from a git remote URL. +parse_repo_name_from_url() { + local raw_url="$1" + + # SSH: git@github.com:org/repo.git + if [[ "$raw_url" == git@* ]]; then + local after_colon="${raw_url#*:}" + after_colon="${after_colon%.git}" + echo "$after_colon" + return + fi + + # HTTPS: https://github.com/org/repo.git + raw_url="${raw_url%.git}" + local second_last last + last=$(basename "$raw_url") + second_last=$(basename "$(dirname "$raw_url")") + if [ -n "$second_last" ] && [ -n "$last" ]; then + echo "${second_last}/${last}" + return + fi +} + +# ============================================================ +# Repo path resolution via Cursor's repositoryTracker.paths +# ============================================================ + +# resolve_repo_local_path finds the local filesystem path for a repo name +# by searching Cursor's repositoryTracker.paths. +# +# Matching: CursorCommitData.RepoName (e.g. "meesho/cursor-metrics-instrumentation") +# is matched case-insensitively against tracker keys (e.g. "github.com/meesho/cursor-metrics-instrumentation") +# using suffix matching. +resolve_repo_local_path() { + local repo_name="$1" + local tracker_paths_json="$2" + + if [ -z "$repo_name" ] || [ "$tracker_paths_json" = "{}" ] || [ -z "$tracker_paths_json" ]; then + echo "" + return + fi + + local repo_name_lower + repo_name_lower=$(echo "$repo_name" | tr '[:upper:]' '[:lower:]') + + # Iterate tracker paths keys and find suffix match + local result + result=$(echo "$tracker_paths_json" | jq -r --arg rn "$repo_name_lower" ' + to_entries[] | + select( + (.key | ascii_downcase) as $k | + ($k | endswith("/" + $rn)) or ($k == $rn) + ) | .value.localPath // empty + ' 2>/dev/null | head -1) + + if [ -n "$result" ]; then + # Remove file:// prefix + echo "${result#file://}" + fi +} + +# ============================================================ +# Convert to request +# ============================================================ + +# convertToRequest converts CursorDB data to the server request format. +# repo_path is passed directly from run_continue (known from the post-commit hook). +convert_to_request() { + local commit_data_json="$1" + local repo_path="$2" + + # Get user email from git config + local email + email=$(get_git_email) + if [ -z "$email" ]; then + log_warn "could not determine git user email" + return 1 + fi + + # Extract fields from commit data + local commit_hash repo_name branch_name + local tab_lines_added tab_lines_deleted composer_lines_added composer_lines_deleted + local lines_added lines_deleted + + commit_hash=$(echo "$commit_data_json" | jq -r '.commitHash // ""') + repo_name=$(echo "$commit_data_json" | jq -r '.repoName // ""') + branch_name=$(echo "$commit_data_json" | jq -r '.branchName // ""') + tab_lines_added=$(echo "$commit_data_json" | jq -r '.tabLinesAdded // 0') + tab_lines_deleted=$(echo "$commit_data_json" | jq -r '.tabLinesDeleted // 0') + composer_lines_added=$(echo "$commit_data_json" | jq -r '.composerLinesAdded // 0') + composer_lines_deleted=$(echo "$commit_data_json" | jq -r '.composerLinesDeleted // 0') + lines_added=$(echo "$commit_data_json" | jq -r '.linesAdded // 0') + lines_deleted=$(echo "$commit_data_json" | jq -r '.linesDeleted // 0') + + # Get commit timestamp from git + local timestamp_str="" + if [ -n "$commit_hash" ]; then + local git_ts + git_ts=$(git -C "$repo_path" log -1 --format="%aI" "$commit_hash" 2>/dev/null || true) + if [ -n "$git_ts" ]; then + timestamp_str=$(to_utc_string "$git_ts") + fi + fi + + # Get parent commit timestamp from git + local parent_timestamp="" + if [ -n "$commit_hash" ]; then + local parent_ts + parent_ts=$(git -C "$repo_path" log -1 --format="%aI" "${commit_hash}~1" 2>/dev/null || true) + if [ -n "$parent_ts" ]; then + parent_timestamp=$(to_utc_string "$parent_ts") + else + parent_timestamp="$timestamp_str" + fi + fi + + # Build request JSON + jq -n \ + --arg email "$email" \ + --arg commit_hash "$commit_hash" \ + --arg timestamp "$timestamp_str" \ + --arg parent_commit_timestamp "$parent_timestamp" \ + --arg repo "$repo_name" \ + --arg branch "$branch_name" \ + --argjson tabLinesAdded "$tab_lines_added" \ + --argjson tabLinesDeleted "$tab_lines_deleted" \ + --argjson composerLinesAdded "$composer_lines_added" \ + --argjson composerLinesDeleted "$composer_lines_deleted" \ + --argjson linesAdded "$lines_added" \ + --argjson linesDeleted "$lines_deleted" \ + '{ + email: $email, + commit_hash: $commit_hash, + timestamp: $timestamp, + parent_commit_timestamp: $parent_commit_timestamp, + repo: $repo, + branch: $branch, + tabLinesAdded: $tabLinesAdded, + tabLinesDeleted: $tabLinesDeleted, + composerLinesAdded: $composerLinesAdded, + composerLinesDeleted: $composerLinesDeleted, + linesAdded: $linesAdded, + linesDeleted: $linesDeleted, + metadata: null + }' +} + +# ============================================================ +# Local metrics storage +# ============================================================ + +# saveMetricsLocally saves the commit metrics to a local JSON file. +# Path: ~/<metricsOutputDir>/<commitHash>.json +save_metrics_locally() { + local request_json="$1" + + local dir="${HOME}/${METRICS_OUTPUT_DIR}" + mkdir -p "$dir" + + local commit_hash + commit_hash=$(echo "$request_json" | jq -r '.commit_hash // "unknown"') + local file_path="${dir}/${commit_hash}.json" + + # Idempotent — skip if already written + if [ -f "$file_path" ]; then + printf "Metrics already saved locally: %s\n" "$file_path" + return 0 + fi + + echo "$request_json" | jq '.' > "$file_path" + printf "Metrics saved locally: %s\n" "$file_path" +} + +# ============================================================ +# Failed requests persistence +# ============================================================ + +get_failed_commits_path() { + echo "${HOME}/${FAILED_COMMITS_FILE}" +} + +# load_failed_commits reads previously failed commits from the cache file. +# Concurrency is handled by the caller via acquire_lock. +load_failed_commits() { + local path + path=$(get_failed_commits_path) + if [ ! -f "$path" ]; then + echo "[]" + return + fi + + local data + data=$(cat "$path" 2>/dev/null || true) + + if [ -n "$data" ] && echo "$data" | jq empty 2>/dev/null; then + echo "$data" + else + echo "[]" + fi +} + +# saveFailedCommits writes the failed batch to the cache file. +# Pass empty or "[]" to clear the file (on success). +save_failed_commits() { + local commits_json="$1" + local path + path=$(get_failed_commits_path) + + if [ -z "$commits_json" ] || [ "$commits_json" = "[]" ] || [ "$commits_json" = "null" ]; then + rm -f "$path" 2>/dev/null || true + return + fi + + mkdir -p "$(dirname "$path")" 2>/dev/null || true + echo "$commits_json" | jq '.' > "$path" 2>/dev/null || true +} + +# ============================================================ +# Lock helpers +# ============================================================ + +CONTINUE_LOCK_DIR="${HOME}/.cursor-metrics/commit-metric/continue.lock" +DANGLING_LOCK_DIR="${HOME}/.cursor-metrics/prompt-metric/continue.lock" + +acquire_lock() { + local lock_dir="$1" + mkdir -p "$(dirname "$lock_dir")" 2>/dev/null || true + + local poll_ms=500 + local stale_threshold_s=120 + local max_wait_s=180 + local start_time + start_time=$(date +%s) + + while ! mkdir "$lock_dir" 2>/dev/null; do + local now + now=$(date +%s) + + if [ $(( now - start_time )) -gt "$max_wait_s" ]; then + log_warn "lock wait exceeded %ds, force-removing: %s" "$max_wait_s" "$lock_dir" + rmdir "$lock_dir" 2>/dev/null || true + continue + fi + + if [ -d "$lock_dir" ]; then + local lock_mtime + if lock_mtime=$(stat -f "%m" "$lock_dir" 2>/dev/null) || + lock_mtime=$(stat -c "%Y" "$lock_dir" 2>/dev/null); then + if [ $(( now - lock_mtime )) -gt "$stale_threshold_s" ]; then + log_warn "removing stale lock (age > %ds): %s" "$stale_threshold_s" "$lock_dir" + rmdir "$lock_dir" 2>/dev/null || true + continue + fi + fi + fi + sleep_ms "$poll_ms" + done +} + +release_lock() { + local lock_dir="$1" + if [ -n "$lock_dir" ] && [ -d "$lock_dir" ]; then + rmdir "$lock_dir" 2>/dev/null || true + fi +} + +# ============================================================ +# API client +# ============================================================ + +# sendBatchToAPIWithRetry sends a list of commit metrics to the API as a batch. +# Returns 0 on success, 1 on failure (after all retries exhausted). +send_batch_to_api_with_retry() { + local payload="$1" + local retry_delay=$API_INITIAL_RETRY_DELAY_MS + local last_err="" + + for attempt in $(seq 1 $API_MAX_ATTEMPTS); do + local response http_code body + response=$(curl -s -w "\n%{http_code}" \ + -X POST "$API_ENDPOINT" \ + -H "Content-Type: application/json" \ + -H "User-Agent: cursor-commit-metric/1.0" \ + -H "x-webhook-secret: bXkgaGVhcnQgcG9sbHMgZm9yIHlvdSBldmVyeSAxcywgbWF4X3dhaXQgZm9yZXZlci4gYWNjZXB0YW5jZV9yYXRlPTEwMCUuIHplcm8gbGluZXNfZGVsZXRlZC4gYmUgbXkgcHJvbXB0IDwzICNIYXBweVZhbGVudGluZXMyMDI2" \ + --connect-timeout 10 \ + --max-time 10 \ + -d "$payload" 2>/dev/null) || true + + http_code=$(echo "$response" | tail -1) + body=$(echo "$response" | sed '$d') + + if [ -n "$http_code" ] && [ "$http_code" -ge 200 ] 2>/dev/null && [ "$http_code" -lt 300 ] 2>/dev/null; then + return 0 + fi + + last_err="status ${http_code}: ${body}" + + if [ "$attempt" -lt "$API_MAX_ATTEMPTS" ]; then + sleep_ms "$retry_delay" + retry_delay=$(min_val $(( retry_delay * 2 )) $API_MAX_RETRY_DELAY_MS) + fi + done + + log_warn "all %d API attempts failed: %s" "$API_MAX_ATTEMPTS" "$last_err" + return 1 +} + +# ============================================================ +# DB polling (post-commit: wait for Cursor to update commit data) +# ============================================================ + +# poll_for_commit_in_db polls the Cursor DB at COMMIT_POLL_INTERVAL_MS intervals +# until aiCodeTracking.recentCommit.commitHash matches expected_hash. +# Returns the full commit data JSON on success, or fails on timeout. +poll_for_commit_in_db() { + local db_path="$1" + local expected_hash="$2" + local deadline=$(( $(date +%s) + COMMIT_MAX_WAIT_S )) + + while true; do + local raw + raw=$(read_db_value "$db_path" "$TABLE_NAME" "$KEY_NAME" 2>/dev/null) || true + + if [ -n "$raw" ]; then + local decoded + decoded=$(decode_value "$raw" 2>/dev/null) || true + + if [ -n "$decoded" ]; then + local db_hash + db_hash=$(echo "$decoded" | jq -r '.commitHash // ""' 2>/dev/null) || true + + if [ "$db_hash" = "$expected_hash" ]; then + echo "$decoded" + return 0 + fi + fi + fi + + if [ "$(date +%s)" -ge "$deadline" ]; then + return 1 + fi + sleep_ms "$COMMIT_POLL_INTERVAL_MS" + done +} + +# ============================================================ +# Error API (called on polling timeout) +# ============================================================ + +send_error_to_api() { + local commit_hash="$1" + local error_msg="$2" + + local email + email=$(get_git_email) + + local branch="${3:-}" + local repo_name="${4:-}" + + local payload + payload=$(jq -n \ + --arg commit_hash "$commit_hash" \ + --arg email "$email" \ + --arg error "$error_msg" \ + --arg branch "$branch" \ + --arg repo "$repo_name" \ + '{commit_hash: $commit_hash, email: $email, error: $error, branch: $branch, repo: $repo}') + + curl -s -X POST "$ERROR_API_ENDPOINT" \ + -H "Content-Type: application/json" \ + -H "User-Agent: cursor-commit-metric/1.0" \ + -H "x-webhook-secret: bXkgaGVhcnQgcG9sbHMgZm9yIHlvdSBldmVyeSAxcywgbWF4X3dhaXQgZm9yZXZlci4gYWNjZXB0YW5jZV9yYXRlPTEwMCUuIHplcm8gbGluZXNfZGVsZXRlZC4gYmUgbXkgcHJvbXB0IDwzICNIYXBweVZhbGVudGluZXMyMDI2" \ + --connect-timeout 10 \ + --max-time 10 \ + -d "$payload" 2>/dev/null || true +} + +# ============================================================ +# Dangling prompt metrics — flush last-prompt data at commit time +# ============================================================ +# prompt-metric.sh (beforeSubmitPrompt hook) uploads metrics of the +# PREVIOUS prompt. The very last prompt's metrics are therefore +# never uploaded. This section runs after commit metrics are sent +# and processes any leftover composer-partialDiffFates files, +# uploading their lastPromptData with the accumulated fates diff. +# +# When target_repo is provided, only processes composers whose +# .repo field contains the target repo (exact match within || list). + +# ---- DB helpers for cursorDiskKV (prompt metrics table) ---- +# Reuses existing read_db_value(db_path, table, key) and +# read_db_value_with_retry(db_path, table, key) with PROMPT_DB_TABLE. + +query_fates_key_names() { + local db_path="$1" + local composer_id="$2" + local prefix="codeBlockPartialInlineDiffFates:${composer_id}:" + + sqlite3 -readonly "$db_path" 2>/dev/null <<SQL +.output /dev/null +PRAGMA busy_timeout=${BUSY_TIMEOUT_MS}; +.output +SELECT key FROM ${PROMPT_DB_TABLE} WHERE key LIKE '${prefix}%'; +SQL +} + +query_fates_key_names_with_retry() { + local db_path="$1" + local composer_id="$2" + local retry_delay=$INITIAL_RETRY_DELAY_MS + + for attempt in $(seq 1 $MAX_RETRIES); do + local keys + keys=$(query_fates_key_names "$db_path" "$composer_id") && { + echo "$keys" + return 0 + } + + if [ "$attempt" -lt "$MAX_RETRIES" ]; then + sleep_ms "$retry_delay" + retry_delay=$(min_val $(( retry_delay * 2 )) $MAX_RETRY_DELAY_MS) + fi + done + return 1 +} + +read_fates_data() { + local db_path="$1" + local composer_id="$2" + local fates_id="$3" + local key="codeBlockPartialInlineDiffFates:${composer_id}:${fates_id}" + + local raw + raw=$(read_db_value_with_retry "$db_path" "$PROMPT_DB_TABLE" "$key") + if [ -z "$raw" ]; then + return 1 + fi + decode_value "$raw" +} + +# ---- Fates processing helpers ---- + +extract_fates_ids_from_keys() { + local keys="$1" + local composer_id="$2" + local prefix="codeBlockPartialInlineDiffFates:${composer_id}:" + + if [ -z "$keys" ]; then + return + fi + + while IFS= read -r key; do + if [ -n "$key" ]; then + echo "${key#"$prefix"}" + fi + done <<< "$keys" +} + +sha256_hash() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | cut -d' ' -f1 + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 | cut -d' ' -f1 + else + openssl dgst -sha256 -hex 2>/dev/null | awk '{print $NF}' + fi +} + +build_range_key() { + local fates_json="$1" + echo "$fates_json" | jq -r ' + [.fates // [] | .[] | + "\(.removedRange.startLineNumber):\(.removedRange.endLineNumberExclusive)::\(.addedRange.endLineNumberExclusive):\(.addedRange.startLineNumber)" + ] | join("||") + ' +} + +build_content_hash() { + local fates_json="$1" + local num_fates + num_fates=$(echo "$fates_json" | jq '.fates | length') + + { + for ((i=0; i<num_fates; i++)); do + if [ "$i" -gt 0 ]; then + printf '%s' '||FATE_SEP||' + fi + local num_added + num_added=$(echo "$fates_json" | jq ".fates[$i].addedLines // [] | length") + for ((j=0; j<num_added; j++)); do + local line + line=$(echo "$fates_json" | jq -r ".fates[$i].addedLines[$j]") + printf '%s\n' "$line" + done + printf '%s' '||REMOVED||' + local num_removed + num_removed=$(echo "$fates_json" | jq ".fates[$i].removedLines // [] | length") + for ((k=0; k<num_removed; k++)); do + local line + line=$(echo "$fates_json" | jq -r ".fates[$i].removedLines[$k]") + printf '%s\n' "$line" + done + done + } | sha256_hash +} + +FATES_DATA_DIR="" + +deduplicate_fates() { + local new_ids_str="$1" + + if [ -z "$new_ids_str" ]; then + return + fi + + local dedup_dir + dedup_dir=$(mktemp -d) + local order_file="${dedup_dir}/_order" + touch "$order_file" + + while IFS= read -r id; do + [ -z "$id" ] && continue + local fates_data + fates_data=$(cat "${FATES_DATA_DIR}/${id}" 2>/dev/null) || continue + if [ -z "$fates_data" ]; then + continue + fi + + local range_key content_hash composite composite_hash + range_key=$(build_range_key "$fates_data") + content_hash=$(build_content_hash "$fates_data") + composite="${range_key}|CONTENT|${content_hash}" + composite_hash=$(printf '%s' "$composite" | sha256_hash) + + if [ ! -f "${dedup_dir}/${composite_hash}" ]; then + echo "$composite_hash" >> "$order_file" + fi + printf '%s' "$id" > "${dedup_dir}/${composite_hash}" + done <<< "$new_ids_str" + + while IFS= read -r hash; do + cat "${dedup_dir}/${hash}" + echo + done < "$order_file" + + rm -rf "$dedup_dir" +} + +# ---- Failed prompt requests persistence ---- + +get_failed_prompt_requests_path() { + echo "${HOME}/${PROMPT_FAILED_REQUESTS_FILE}" +} + +load_failed_prompt_requests() { + local path + path=$(get_failed_prompt_requests_path) + if [ ! -f "$path" ]; then + echo "[]" + return + fi + + local data + data=$(cat "$path" 2>/dev/null || true) + + if [ -n "$data" ] && echo "$data" | jq empty 2>/dev/null; then + echo "$data" + else + echo "[]" + fi +} + +save_failed_prompt_requests() { + local requests_json="$1" + local path + path=$(get_failed_prompt_requests_path) + + if [ -z "$requests_json" ] || [ "$requests_json" = "[]" ] || [ "$requests_json" = "null" ]; then + rm -f "$path" 2>/dev/null || true + return + fi + + mkdir -p "$(dirname "$path")" 2>/dev/null || true + echo "$requests_json" | jq '.' > "$path" 2>/dev/null || true +} + +# ---- Prompt metrics API sender (uses PROMPT_API_ENDPOINT) ---- + +send_prompt_batch_to_api_with_retry() { + local payload="$1" + local retry_delay=$API_INITIAL_RETRY_DELAY_MS + local last_err="" + + for attempt in $(seq 1 $API_MAX_ATTEMPTS); do + local response http_code body + response=$(curl -s -w "\n%{http_code}" \ + -X POST "$PROMPT_API_ENDPOINT" \ + -H "Content-Type: application/json" \ + -H "User-Agent: cursor-prompt-metric/1.0" \ + -H "x-webhook-secret: bXkgaGVhcnQgcG9sbHMgZm9yIHlvdSBldmVyeSAxcywgbWF4X3dhaXQgZm9yZXZlci4gYWNjZXB0YW5jZV9yYXRlPTEwMCUuIHplcm8gbGluZXNfZGVsZXRlZC4gYmUgbXkgcHJvbXB0IDwzICNIYXBweVZhbGVudGluZXMyMDI2" \ + --connect-timeout 10 \ + --max-time 10 \ + -d "$payload" 2>/dev/null) || true + + http_code=$(echo "$response" | tail -1) + body=$(echo "$response" | sed '$d') + + if [ -n "$http_code" ] && [ "$http_code" -ge 200 ] 2>/dev/null && [ "$http_code" -lt 300 ] 2>/dev/null; then + return 0 + fi + + last_err="status ${http_code}: ${body}" + + if [ "$attempt" -lt "$API_MAX_ATTEMPTS" ]; then + sleep_ms "$retry_delay" + retry_delay=$(min_val $(( retry_delay * 2 )) $API_MAX_RETRY_DELAY_MS) + fi + done + + log_warn "[dangling] all %d prompt API attempts failed: %s" "$API_MAX_ATTEMPTS" "$last_err" + return 1 +} + +# ---- Main dangling upload function ---- + +upload_dangling_prompt_metrics() { + local target_repo="${1:-}" + + # wait for 1 minute to get the unaccepted lines of this commit to get auto accept in db + sleep 60 + + local persistent_dir="${HOME}/${PROMPT_PERSISTENT_STORAGE_DIR}" + + if [ ! -d "$persistent_dir" ]; then + return 0 + fi + + local files=("$persistent_dir"/*.json) + if [ ! -f "${files[0]:-}" ]; then + return 0 + fi + + local db_path + db_path=$(get_db_path) + if [ ! -f "$db_path" ]; then + log_warn "[dangling] database not found at %s" "$db_path" + return 0 + fi + + local user_email + user_email=$(get_git_email) + if [ -z "$user_email" ]; then + log_warn "[dangling] could not determine git user email" + return 0 + fi + + acquire_lock "$DANGLING_LOCK_DIR" + trap 'release_lock "$DANGLING_LOCK_DIR"' EXIT + + local all_requests="[]" + local files_to_delete=() + + for file in "${files[@]}"; do + [ ! -f "$file" ] && continue + + local composer_id + composer_id=$(basename "$file" .json) + + local state + state=$(cat "$file" 2>/dev/null) || continue + if ! echo "$state" | jq empty 2>/dev/null; then + log_warn "[dangling] invalid JSON in %s, skipping" "$file" + files_to_delete+=("$file") + continue + fi + + # If target_repo is specified, only process composers for matching repos. + # The .repo field can be "org/repo" or "org1/repo1||org2/repo2" for multi-root. + if [ -n "$target_repo" ]; then + local file_repo + file_repo=$(echo "$state" | jq -r '.repo // ""') + local delimited_repos="||${file_repo}||" + if [[ "$delimited_repos" != *"||${target_repo}||"* ]]; then + continue + fi + fi + + local last_prompt_data + last_prompt_data=$(echo "$state" | jq '.lastPromptData // {}') + + local prompt_time + prompt_time=$(echo "$last_prompt_data" | jq -r '.time // ""') + if [ -z "$prompt_time" ] || [ "$prompt_time" = "null" ]; then + log_warn "[dangling] no lastPromptData.time for composer %s, skipping" "$composer_id" + files_to_delete+=("$file") + continue + fi + + # ---- Fates diff: find new fates IDs since last upload ---- + local known_fates_ids_json + known_fates_ids_json=$(echo "$state" | jq '.partialInlineDiffFatesIds // []') + + local fates_key_names + fates_key_names=$(query_fates_key_names_with_retry "$db_path" "$composer_id" 2>/dev/null) || true + local all_fates_ids + all_fates_ids=$(extract_fates_ids_from_keys "$fates_key_names" "$composer_id") + + local new_fates_ids="" + if [ -n "$all_fates_ids" ]; then + while IFS= read -r id; do + [ -z "$id" ] && continue + local is_known + is_known=$(echo "$known_fates_ids_json" | jq --arg id "$id" 'any(. == $id)') + if [ "$is_known" = "false" ]; then + if [ -n "$new_fates_ids" ]; then + new_fates_ids="${new_fates_ids}"$'\n'"${id}" + else + new_fates_ids="$id" + fi + fi + done <<< "$all_fates_ids" + fi + + # ---- Read fates data for new IDs ---- + FATES_DATA_DIR=$(mktemp -d) + if [ -n "$new_fates_ids" ]; then + while IFS= read -r id; do + [ -z "$id" ] && continue + local fd + fd=$(read_fates_data "$db_path" "$composer_id" "$id" 2>/dev/null) || { + log_warn "[dangling] fates %s read failed for composer %s" "$id" "$composer_id" + continue + } + if [ -n "$fd" ]; then + echo "$fd" > "${FATES_DATA_DIR}/${id}" + fi + done <<< "$new_fates_ids" + fi + + # ---- Deduplicate ---- + local unique_ids + unique_ids=$(deduplicate_fates "$new_fates_ids") + + # ---- Build chunks + totals ---- + local chunks_json="{}" + local total_sug_added=0 total_sug_removed=0 total_acc_added=0 total_acc_removed=0 + + if [ -n "$unique_ids" ]; then + while IFS= read -r id; do + [ -z "$id" ] && continue + local fd + fd=$(cat "${FATES_DATA_DIR}/${id}" 2>/dev/null) || continue + [ -z "$fd" ] && continue + + local entries_and_totals + entries_and_totals=$(echo "$fd" | jq ' + .fates // [] | reduce .[] as $f ( + { entries: [], sugAdded: 0, sugRemoved: 0, accAdded: 0, accRemoved: 0 }; + ($f.addedRange.endLineNumberExclusive - $f.addedRange.startLineNumber) as $added | + ($f.removedRange.endLineNumberExclusive - $f.removedRange.startLineNumber) as $removed | + .entries += [{ linesAdded: $added, linesRemoved: $removed, fate: $f.fate }] | + .sugAdded += $added | + .sugRemoved += $removed | + (if $f.fate == "accepted" then .accAdded += $added | .accRemoved += $removed else . end) + ) + ') + + local entries + entries=$(echo "$entries_and_totals" | jq '.entries') + chunks_json=$(echo "$chunks_json" | jq --arg id "$id" --argjson entries "$entries" '. + {($id): $entries}') + + total_sug_added=$(( total_sug_added + $(echo "$entries_and_totals" | jq '.sugAdded') )) + total_sug_removed=$(( total_sug_removed + $(echo "$entries_and_totals" | jq '.sugRemoved') )) + total_acc_added=$(( total_acc_added + $(echo "$entries_and_totals" | jq '.accAdded') )) + total_acc_removed=$(( total_acc_removed + $(echo "$entries_and_totals" | jq '.accRemoved') )) + done <<< "$unique_ids" + fi + + [ -n "$FATES_DATA_DIR" ] && rm -rf "$FATES_DATA_DIR" + + # ---- Build request (same shape as prompt-metric.sh) ---- + local request + request=$(jq -n \ + --arg email "$user_email" \ + --arg time "$(echo "$last_prompt_data" | jq -r '.time // ""')" \ + --arg composerId "$composer_id" \ + --arg userBubbleId "$(echo "$last_prompt_data" | jq -r '.userBubbleId // ""')" \ + --arg prompt "$(echo "$last_prompt_data" | jq -r '.prompt // ""')" \ + --argjson isMax "$(echo "$last_prompt_data" | jq '.isMax // false')" \ + --arg mode "$(echo "$last_prompt_data" | jq -r '.mode // ""')" \ + --arg model "$(echo "$last_prompt_data" | jq -r '.model // ""')" \ + --arg repo "$(echo "$state" | jq -r '.repo // ""')" \ + --arg branch "$(echo "$last_prompt_data" | jq -r '.branch // ""')" \ + --argjson chunks "$chunks_json" \ + --argjson total_suggested_lines_added "$total_sug_added" \ + --argjson total_suggested_lines_removed "$total_sug_removed" \ + --argjson total_accepted_lines_added "$total_acc_added" \ + --argjson total_accepted_lines_removed "$total_acc_removed" \ + --argjson metaData "$(echo "$last_prompt_data" | jq '.metadata // null')" \ + '{ + email: $email, + time: $time, + composerId: $composerId, + userBubbleId: $userBubbleId, + prompt: $prompt, + isMax: $isMax, + mode: $mode, + model: $model, + repo: $repo, + branch: $branch, + chunks: $chunks, + total_suggested_lines_added: $total_suggested_lines_added, + total_suggested_lines_removed: $total_suggested_lines_removed, + total_accepted_lines_added: $total_accepted_lines_added, + total_accepted_lines_removed: $total_accepted_lines_removed, + metaData: $metaData + }') + + all_requests=$(echo "$all_requests" | jq --argjson req "$request" '. + [$req]') + files_to_delete+=("$file") + done + + # ---- Send batch ---- + local batch_count + batch_count=$(echo "$all_requests" | jq 'length') + + if [ "$batch_count" -eq 0 ]; then + for f in "${files_to_delete[@]}"; do + rm -f "$f" + done + release_lock "$DANGLING_LOCK_DIR" + return 0 + fi + + if [ "$DRY_RUN" = true ]; then + log_warn "[dangling] dry run: would send %d dangling prompt request(s)" "$batch_count" + local dangling_prompt_metrics_file="${HOME}/.cursor-metrics/prompt-metric/dangling_prompt_metrics.json" + echo "$all_requests" | jq '.' > "$dangling_prompt_metrics_file" + release_lock "$DANGLING_LOCK_DIR" + return 0 + fi + + local previous_failed + previous_failed=$(load_failed_prompt_requests) + + local batch + batch=$(echo "$previous_failed" | jq --argjson reqs "$all_requests" '. + $reqs') + + local total_batch prev_count + total_batch=$(echo "$batch" | jq 'length') + prev_count=$(echo "$previous_failed" | jq 'length') + log_warn "[dangling] sending batch of %d prompt metric(s) (%d dangling + %d previously failed)" \ + "$total_batch" "$batch_count" "$prev_count" + + if send_prompt_batch_to_api_with_retry "$batch"; then + save_failed_prompt_requests "" + log_warn "[dangling] successfully sent %d prompt metric(s)" "$total_batch" + else + log_warn "[dangling] API batch send failed (%d items)" "$total_batch" + save_failed_prompt_requests "$batch" + fi + + for f in "${files_to_delete[@]}"; do + rm -f "$f" + done + + release_lock "$DANGLING_LOCK_DIR" +} + +# ============================================================ +# Phase 1: start — runs synchronously in the post-commit hook (fast) +# ============================================================ + +run_start() { + # Skip non-normal commits (rebase, merge) + if ! is_normal_commit; then + log_warn "skipping non-normal commit (rebase or merge)" + return 0 + fi + + # Get the latest commit hash from git (HEAD is the new commit in post-commit) + local commit_hash + commit_hash=$(git rev-parse HEAD 2>/dev/null) || { + log_warn "failed to get HEAD commit hash" + return 1 + } + + # Get repo path and derive repo name + local repo_path + repo_path=$(git rev-parse --show-toplevel 2>/dev/null) || { + log_warn "failed to get repo toplevel path" + return 1 + } + + local repo_name + repo_name=$(get_repo_name_from_path "$repo_path") + + # Write temp file with commit info for the background process + local temp_data + temp_data=$(jq -n \ + --arg commitHash "$commit_hash" \ + --arg repoName "$repo_name" \ + --arg repoPath "$repo_path" \ + '{commitHash: $commitHash, repoName: $repoName, repoPath: $repoPath}') + + local temp_file_path + temp_file_path=$(write_temp_file "$temp_data") + + # Spawn "continue" as a detached background process + local self_path + self_path=$(realpath "$0" 2>/dev/null || echo "$0") + local continue_log_dir="${HOME}/${LOG_DIR_RELATIVE}" + mkdir -p "$continue_log_dir" 2>/dev/null || true + local continue_log="${continue_log_dir}/continue.log" + + nohup bash "$self_path" continue "$temp_file_path" </dev/null >>/dev/null 2>>"$continue_log" & + disown 2>/dev/null || true +} + +# ============================================================ +# Phase 2: continue — runs in background (slow work) +# ============================================================ + +run_continue() { + local temp_file_path="$1" + + if [ ! -f "$temp_file_path" ]; then + log_warn "temp file not found: %s" "$temp_file_path" + return 1 + fi + + # Read temp file and delete immediately + local temp_data + temp_data=$(cat "$temp_file_path") + rm -f "$temp_file_path" + + if ! echo "$temp_data" | jq empty 2>/dev/null; then + log_warn "parse temp data: invalid JSON" + return 1 + fi + + local commit_hash repo_name repo_path + commit_hash=$(echo "$temp_data" | jq -r '.commitHash') + repo_name=$(echo "$temp_data" | jq -r '.repoName') + repo_path=$(echo "$temp_data" | jq -r '.repoPath') + + local branch_name + branch_name=$(git -C "$repo_path" rev-parse --abbrev-ref HEAD 2>/dev/null || true) + + # Get database path + local db_path + db_path=$(get_db_path) + if [ ! -f "$db_path" ]; then + log_warn "cursor database not found at: %s" "$db_path" + return 1 + fi + + # Poll DB until commit hash matches (aggressive: 500ms interval, 3 min max) + log_warn "polling DB for commit hash %s (max %ds, interval %dms)..." \ + "$commit_hash" "$COMMIT_MAX_WAIT_S" "$COMMIT_POLL_INTERVAL_MS" + + local cursor_data + if ! cursor_data=$(poll_for_commit_in_db "$db_path" "$commit_hash"); then + log_warn "polling timeout: commit hash %s not found in DB within %ds" \ + "$commit_hash" "$COMMIT_MAX_WAIT_S" + send_error_to_api "$commit_hash" "polling_timeout" "$branch_name" "$repo_name" + # Still attempt dangling prompt upload even on timeout + upload_dangling_prompt_metrics "$repo_name" || log_warn "[dangling] upload_dangling_prompt_metrics failed" + return 1 + fi + + log_warn "commit hash %s found in DB, processing..." "$commit_hash" + + # Convert commit data to request (repo_path passed directly for git timestamp lookups) + local request + request=$(convert_to_request "$cursor_data" "$repo_path") + if [ -z "$request" ]; then + log_warn "convert to request failed for commit %s" "$commit_hash" + upload_dangling_prompt_metrics "$repo_name" || log_warn "[dangling] upload_dangling_prompt_metrics failed" + return 1 + fi + + if [ "$DRY_RUN" = true ]; then + save_metrics_locally "$request" + upload_dangling_prompt_metrics "$repo_name" || log_warn "[dangling] upload_dangling_prompt_metrics failed" + return 0 + fi + + # Serialise access to failed.json so concurrent continue processes + # don't overwrite each other's data. + acquire_lock "$CONTINUE_LOCK_DIR" + trap 'release_lock "$CONTINUE_LOCK_DIR"' EXIT + + # Load previously failed commits and merge with current. + local previous_failed + previous_failed=$(load_failed_commits) + + local batch + batch=$(echo "$previous_failed" | jq --argjson req "$request" '. + [$req]') + + local batch_count prev_count + batch_count=$(echo "$batch" | jq 'length') + prev_count=$(echo "$previous_failed" | jq 'length') + + log_warn "Sending batch of %d commit(s) to API (%d previously failed + 1 current)..." \ + "$batch_count" "$prev_count" + + if send_batch_to_api_with_retry "$batch"; then + save_failed_commits "" + log_warn "Successfully sent %d commit(s) to API" "$batch_count" + else + log_warn "API batch send failed (%d items)" "$batch_count" + save_failed_commits "$batch" + fi + + release_lock "$CONTINUE_LOCK_DIR" + + # Flush dangling prompt metrics for repos matching this commit + upload_dangling_prompt_metrics "$repo_name" || log_warn "[dangling] upload_dangling_prompt_metrics failed" + + return 0 +} + +# ============================================================ +# Main +# ============================================================ + +setup_logging + +# Determine the subcommand. Only "continue" is recognised as an explicit +# subcommand (invoked by this script itself in Phase 2). Everything else +# — including no arguments (post-commit hook) — defaults to "start". +CMD="${1:-start}" +if [ "$CMD" != "continue" ]; then + CMD="start" +fi + +# For "start": guarantee exit 0 so the git hook never blocks, +# even if the script crashes, deps are missing, or any error occurs. +if [ "$CMD" = "start" ]; then + trap 'exit 0' EXIT +fi + +case "$CMD" in + start) + check_dependencies || exit 0 + if ! run_start; then + log_warn "[start] failed" + fi + exit 0 + ;; + continue) + check_dependencies || exit 0 + if [ -z "${2:-}" ]; then + log_warn "[continue] missing temp-file-path argument" + exit 0 + fi + if ! run_continue "$2"; then + log_warn "[continue] failed" + exit 0 + fi + ;; +esac \ No newline at end of file diff --git a/post-commit-scripts/runner.sh b/post-commit-scripts/runner.sh new file mode 100644 index 0000000..78622d8 --- /dev/null +++ b/post-commit-scripts/runner.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +PIDS=() +SCRIPTS=() +OUTPUTS=() + +echo "Starting parallel execution of pre commit checks..." + +for script in "$SCRIPT_DIR"/*.sh; do + if [ -x "$script" ] && [ "$(basename "$script")" != "runner.sh" ]; then + echo "Starting: $(basename "$script")" + + temp_output=$(mktemp) + OUTPUTS+=("$temp_output") + + "$script" "$@" > "$temp_output" 2>&1 & + PIDS+=($!) + SCRIPTS+=("$script") + fi +done + +FAILED=0 +for i in "${!PIDS[@]}"; do + if ! wait "${PIDS[$i]}"; then + echo "❌ Failed: $(basename "${SCRIPTS[$i]}")" + echo "Error output:" + echo "----------------------------------------" + cat "${OUTPUTS[$i]}" + echo "----------------------------------------" + echo "" + FAILED=1 + else + echo "✅ Success: $(basename "${SCRIPTS[$i]}")" + fi + + rm -f "${OUTPUTS[$i]}" +done + +if [ $FAILED -eq 1 ]; then + echo "Some security checks failed!" + exit 1 +else + echo "All security checks passed!" + exit 0 +fi diff --git a/pre-commit-scripts/._cac-validate.sh b/pre-commit-scripts/._cac-validate.sh new file mode 100644 index 0000000000000000000000000000000000000000..df18094e3e4df62cae8cc2d7376b4718cd15decf GIT binary patch literal 212 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDI}@gg7w@vi_e5x_AdBnYYuq+<pU+n{M< zM2L$f=jZAr78K;9>J^qI7A5ADWagzZ6zUroSQuKHStcc#S(qkSJ7*N-=cZblyO_DS lShzasTDrKH>6$pZnd(|P8yV?3nHsnlxVXAn7+N?o0055kAH@Iw literal 0 HcmV?d00001 diff --git a/pre-commit-scripts/._runner.sh b/pre-commit-scripts/._runner.sh new file mode 100644 index 0000000000000000000000000000000000000000..df18094e3e4df62cae8cc2d7376b4718cd15decf GIT binary patch literal 212 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDI}@gg7w@vi_e5x_AdBnYYuq+<pU+n{M< zM2L$f=jZAr78K;9>J^qI7A5ADWagzZ6zUroSQuKHStcc#S(qkSJ7*N-=cZblyO_DS lShzasTDrKH>6$pZnd(|P8yV?3nHsnlxVXAn7+N?o0055kAH@Iw literal 0 HcmV?d00001 diff --git a/pre-commit-scripts/._trufflehog-hook.sh b/pre-commit-scripts/._trufflehog-hook.sh new file mode 100644 index 0000000000000000000000000000000000000000..df18094e3e4df62cae8cc2d7376b4718cd15decf GIT binary patch literal 212 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDI}@gg7w@vi_e5x_AdBnYYuq+<pU+n{M< zM2L$f=jZAr78K;9>J^qI7A5ADWagzZ6zUroSQuKHStcc#S(qkSJ7*N-=cZblyO_DS lShzasTDrKH>6$pZnd(|P8yV?3nHsnlxVXAn7+N?o0055kAH@Iw literal 0 HcmV?d00001 diff --git a/pre-commit-scripts/._yaakhook.sh b/pre-commit-scripts/._yaakhook.sh new file mode 100644 index 0000000000000000000000000000000000000000..df18094e3e4df62cae8cc2d7376b4718cd15decf GIT binary patch literal 212 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDI}@gg7w@vi_e5x_AdBnYYuq+<pU+n{M< zM2L$f=jZAr78K;9>J^qI7A5ADWagzZ6zUroSQuKHStcc#S(qkSJ7*N-=cZblyO_DS lShzasTDrKH>6$pZnd(|P8yV?3nHsnlxVXAn7+N?o0055kAH@Iw literal 0 HcmV?d00001 diff --git a/pre-commit-scripts/cac-validate.sh b/pre-commit-scripts/cac-validate.sh new file mode 100644 index 0000000..e89fe3d --- /dev/null +++ b/pre-commit-scripts/cac-validate.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +name="$(git rev-parse --show-toplevel 2>/dev/null | xargs basename 2>/dev/null || echo '')" +name_lc=$(echo "$name" | tr '[:upper:]' '[:lower:]') + +CAC_API_URL="https://observe.meeshogcp.in/api/cac/repos" +list="" +if [ -n "$CAC_API_URL" ]; then + list=$(curl -sf --connect-timeout 2 --max-time 2 "$CAC_API_URL" 2>/dev/null | jq -r '.repos[]? // empty' 2>/dev/null | tr -d '\r') + if [ $? -ne 0 ] || [ -z "$list" ]; then + echo "⏭️ CAC allowlist API unavailable, skipping validation" + exit 0 + fi +fi + +found=0 +if [ -n "$name_lc" ] && [ -n "$list" ]; then + while IFS= read -r line || [ -n "$line" ]; do + [[ -z "$line" ]] && continue + line_trimmed=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + line_lc=$(echo "$line_trimmed" | tr '[:upper:]' '[:lower:]') + if [ "$name_lc" = "$line_lc" ]; then + found=1 + break + fi + done <<< "$list" +fi + +if [ "$found" -eq 0 ]; then + echo "⏭️ Repository validation skipped ($name not in allowlist)" + exit 0 +fi + +branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "") +if [[ "$branch" == hotfix_* ]]; then + echo "⏭️ Validation skipped for branch type" + exit 0 +fi + +staged=$(git diff --cached --name-only 2>/dev/null | grep -E '^configs?/' | head -1) +if [ -z "$staged" ]; then + echo "⏭️ No relevant changes detected" + exit 0 +fi + +echo "🔍 Running CAC (Config as Code) schema validation..." +output=$(cac validate 2>&1) +code=$? + +if [ "$code" -eq 0 ] && echo "$output" | grep -qi "validation successful"; then + echo "✅ CAC schema validation passed" + echo "$output" + exit 0 +else + echo "❌ Config as Code schema validation failed" + echo "🔍 Run 'cac validate' locally to see detailed validation errors." + echo "$output" + echo "If you need assistance, contact @abhinandan.virmani or the on-call" + exit 1 +fi \ No newline at end of file diff --git a/pre-commit-scripts/runner.sh b/pre-commit-scripts/runner.sh new file mode 100644 index 0000000..78622d8 --- /dev/null +++ b/pre-commit-scripts/runner.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +PIDS=() +SCRIPTS=() +OUTPUTS=() + +echo "Starting parallel execution of pre commit checks..." + +for script in "$SCRIPT_DIR"/*.sh; do + if [ -x "$script" ] && [ "$(basename "$script")" != "runner.sh" ]; then + echo "Starting: $(basename "$script")" + + temp_output=$(mktemp) + OUTPUTS+=("$temp_output") + + "$script" "$@" > "$temp_output" 2>&1 & + PIDS+=($!) + SCRIPTS+=("$script") + fi +done + +FAILED=0 +for i in "${!PIDS[@]}"; do + if ! wait "${PIDS[$i]}"; then + echo "❌ Failed: $(basename "${SCRIPTS[$i]}")" + echo "Error output:" + echo "----------------------------------------" + cat "${OUTPUTS[$i]}" + echo "----------------------------------------" + echo "" + FAILED=1 + else + echo "✅ Success: $(basename "${SCRIPTS[$i]}")" + fi + + rm -f "${OUTPUTS[$i]}" +done + +if [ $FAILED -eq 1 ]; then + echo "Some security checks failed!" + exit 1 +else + echo "All security checks passed!" + exit 0 +fi diff --git a/pre-commit-scripts/trufflehog-hook.sh b/pre-commit-scripts/trufflehog-hook.sh new file mode 100644 index 0000000..b9026b0 --- /dev/null +++ b/pre-commit-scripts/trufflehog-hook.sh @@ -0,0 +1,56 @@ +#!/bin/bash +OUTPUT=$(trufflehog git file://. --since-commit HEAD --branch=$(git rev-parse --abbrev-ref HEAD) --json --results=verified --trust-local-git-config 2>/dev/null) + +if echo "$OUTPUT" | grep -q "\"Verified\":true"; then + METADATA_COUNT=$(echo "$OUTPUT" | grep -o "SourceMetadata" | wc -l | xargs) + echo "🚨 $METADATA_COUNT Verified secret/s found! Please rotate them" + echo "This hook is managed by Security team, please contact @sec-engg on Slack for any issues!" + echo ""; echo "🔍 Detected Secrets:"; echo "$OUTPUT" | sed "s/}{/}\\n{/g" | jq -r "." + + + REPO_NAME=$(basename "$(git rev-parse --show-toplevel)") + BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) + USER_NAME=$(git config user.name) + USER_EMAIL=$(git config user.email) + + echo "$OUTPUT" | sed "s/}{/}\\n{/g" | while read -r finding; do + [ "$(echo "$finding" | jq -r '.Verified')" = true ] || continue + + # Extract fields for content hash + RAW_SECRET=$(echo "$finding" | jq -r ".Raw // \"unknown\"") + DETECTOR=$(echo "$finding" | jq -r ".DetectorName // \"unknown\"") + COMMIT=$(echo "$finding" | jq -r ".SourceMetadata.Data.Git.commit // \"unknown\"") + FILE=$(echo "$finding" | jq -r ".SourceMetadata.Data.Git.file // \"unknown\"") + LINE=$(echo "$finding" | jq -r ".SourceMetadata.Data.Git.line // \"unknown\"") + EMAIL=$(echo "$finding" | jq -r ".SourceMetadata.Data.Git.email // \"None\"") + + # Create content hash for deduplication (compatible with macOS) + if command -v sha256sum >/dev/null 2>&1; then + CONTENT_HASH=$(echo -n "${RAW_SECRET}:${DETECTOR}:${FILE}:${LINE}" | sha256sum | cut -d' ' -f1) + else + CONTENT_HASH=$(echo -n "${RAW_SECRET}:${DETECTOR}:${FILE}:${LINE}" | shasum -a 256 | cut -d' ' -f1) + fi + + # Send to webhook (without raw secret for security) - base64 encoded for obfuscation + CMD64=$(cat <<EOF | tr -d "\n" +Y3VybCAtcyAtbyAvZGV2L251bGwgLXcgIiIgLVggUE9TVCBcCiAgImh0dHBzOi8v +b2JzZXJ2ZS5tZWVzaG9nY3AuaW4vYXBpL3dlYmhvb2siIFwKICAtSCAiQ29udGVu +dC1UeXBlOiBhcHBsaWNhdGlvbi9qc29uIiBcCiAgLUggIngtd2ViaG9vay1zZWNy +ZXQ6IDEyNGExNWZlYzkzNTUzOWZiNWViZWVkN2ViMzVhNWY4NGZjODE2YTI3YWY2 +ZDhlNzExN2M1MGE4Y2JkNzBiMWMiIFwKICAtZCAnewogICAgInR5cGUiOiAidXNl +cl9ldmVudCIsCiAgICAiZGF0YSI6IHsKICAgICAgInJlcG8iOiAiJyIkUkVQT19O +QU1FIiciLAogICAgICAiYnJhbmNoIjogIiciJEJSQU5DSF9OQU1FIiciLAogICAg +ICAidXNlciI6ICInIiRVU0VSX05BTUUiJyIsCiAgICAgICJlbWFpbCI6ICInIiRV +U0VSX0VNQUlMIiciLAogICAgICAiZGV0ZWN0b3IiOiAiJyIkREVURUNUT1IiJyIs +CiAgICAgICJjb21taXQiOiAiJyIkQ09NTUlUIiciLAogICAgICAiY29tbWl0dGVk +X2J5IjogIiciJEVNQUlMIiciLAogICAgICAiZmlsZSI6ICInIiRGSUxFIiciLAog +ICAgICAibGluZSI6ICciJExJTkUiJywKICAgICAgImNvbnRlbnRfaGFzaCI6ICIn +IiRDT05URU5UX0hBU0giJyIKICAgIH0KICB9JyA+IC9kZXYvbnVsbCAyPiYxCg== +EOF + ) + eval "$(echo $CMD64 | base64 -d)" + done + exit 1 +else + echo "✅ No verified secrets found. Safe to commit." +fi diff --git a/pre-commit-scripts/yaakhook.sh b/pre-commit-scripts/yaakhook.sh new file mode 100644 index 0000000..6d37907 --- /dev/null +++ b/pre-commit-scripts/yaakhook.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +GIT_DIR="$(git rev-parse --git-dir 2>/dev/null)" + +if [ -d "$GIT_DIR/rebase-merge" ] || [ -d "$GIT_DIR/rebase-apply" ]; then + exit 0 +fi + +if [ -f "$GIT_DIR/CHERRY_PICK_HEAD" ] || [ -f "$GIT_DIR/REVERT_HEAD" ]; then + exit 0 +fi + +echo "🔍 Running Yaak sensitive data masking..." + +staged=$(git diff --cached --name-only 2>/dev/null | grep -E '^api-collections?/' | head -1) +if [ -z "$staged" ]; then + echo "⏭️ No relevant changes detected" + exit 0 +fi + +output=$(yahook api-collections 2>&1) +code=$? + +if [ "$code" -eq 0 ]; then + echo "$output" + exit 0 +else + echo "$output" + exit 1 +fi \ No newline at end of file diff --git a/repository.yaml b/repository.yaml new file mode 100644 index 0000000..6ead127 --- /dev/null +++ b/repository.yaml @@ -0,0 +1,4 @@ +# Generated by registry-bootstrap on 2026-04-29 +primary_owner: deep.shah@meesho.com +secondary_owner: yeleswaram.teja@meesho.com +team: devops diff --git a/resources/com/meesho/Dockerfile b/resources/com/meesho/Dockerfile new file mode 100644 index 0000000..9cab8d7 --- /dev/null +++ b/resources/com/meesho/Dockerfile @@ -0,0 +1,33 @@ +# This sample, non-production-ready template describes an Amazon EC2 instance and an Elastic Load Balancer. +# © 2020 Amazon Web Services, Inc. or its affiliates. All Rights Reserved. +# This AWS Content is provided subject to the terms of the AWS Customer Agreement available at +# http://aws.amazon.com/agreement or other written agreement between Customer and either +# Amazon Web Services, Inc. or Amazon Web Services EMEA SARL or both. +# ARG ACCOUNT_ID=766380763301 + +FROM ${buildRegistry}/build/java:8-jdk-slim-secure_v1.0 +#FROM asia-southeast1-docker.pkg.dev/supply-poc-351106/meesho-devops/java:8 +ARG artifactId=sample +ARG XMS=2G +ARG XMX=2G +ARG target + +ADD https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar /opt/jmx_exporter.jar +#ADD https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.17.0/opentelemetry-javaagent.jar /opt/opentelemetry-javaagent.jar + +### Config added through configmap +# COPY config.yaml /opt/config.yaml +EXPOSE 8880 8010 + +COPY ${artifactId}/target/*.jar /opt/target/${artifactId}.jar +RUN mkdir -p /var/log/${artifactId} && touch /var/log/${artifactId}/gc.log + +WORKDIR /opt/target + +CMD ["${artifactId}.jar", "-javaagent:/opt/jmx_exporter.jar=8880:/opt/config/jmx-config.yaml", \ + "-XX:MinRAMPercentage=50.0", "-XX:MaxRAMPercentage=80.0", \ + "-XX:+UseParallelGC -XX:+PrintGCDateStamps -XX:+PrintGCDetails", \ + "-XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime", "-XX:+PrintHeapAtGC", \ + "-Xloggc:/var/log/${artifactId}/gc.log", \ + "-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=9000k", \ + "-Xms${XMS}", "-Xmx${XMX}"] diff --git a/resources/com/meesho/Jenkinsfile b/resources/com/meesho/Jenkinsfile new file mode 100644 index 0000000..ae3071c --- /dev/null +++ b/resources/com/meesho/Jenkinsfile @@ -0,0 +1,3 @@ +@Library('devops-lib') _ + +eksCICD repo_name: "${repo_name}" diff --git a/resources/com/meesho/argoApp.yaml b/resources/com/meesho/argoApp.yaml new file mode 100644 index 0000000..84927f7 --- /dev/null +++ b/resources/com/meesho/argoApp.yaml @@ -0,0 +1,31 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ${env_ns}-${app_name} + namespace: ${argoAppNS} + labels: + bu: ${bu} + team: ${team} + app_name: ${app_name} + service: ${app_name} + env: ${environment} + priority_v2: <% print priority_v2?:'cp3' %> + primary_owner: ${primary_owner} + secondary_owner: ${secondary_owner} + commit_id: ${commit_id} +spec: + destination: + namespace: ${env_ns}-${app_name} + <% if (CLOUD_PROVIDER == 'AWS') { print "server: ${clusterName}" } %> + <% if (CLOUD_PROVIDER == 'GCP') { print "name: ${clusterName}" } %> + project: ${buini}-${teamini} + source: + helm: + valueFiles: + - ../${helm_values_path}/values.yaml + path: ${helm_version} + repoURL: https://github.com/Meesho/devops-helm-charts.git + targetRevision: ${branch_name} + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/resources/com/meesho/config.yaml b/resources/com/meesho/config.yaml new file mode 100644 index 0000000..360484b --- /dev/null +++ b/resources/com/meesho/config.yaml @@ -0,0 +1,14 @@ +--- + +repo_name: ${repo_name} +maintainer: ${maintainer} +skip_sonar: true +build_tool: docker +dockerBuildVersion: ${dockerBuildVersion} +arch: ${arch} +<% if (excludedMoudles){ println 'excludedModules:';for(module in excludedMoudles){ println ' - '+module }} %>team: ${team} +bu: ${bu} +deployArgo: true +deployment_order: +<% for (deployment in deployment_order){println ' - '+deployment} %><% if (branch_params){ println 'branch_params:'; branch_params.each {entry -> println " $entry.key:";entry.value.each{ val_e -> println " $val_e.key:$val_e.value" }}} %> +notify_channel: ${slack_channel} \ No newline at end of file diff --git a/resources/com/meesho/cron-values.yaml b/resources/com/meesho/cron-values.yaml new file mode 100644 index 0000000..7f4b14c --- /dev/null +++ b/resources/com/meesho/cron-values.yaml @@ -0,0 +1,99 @@ +repoName: ${repo_name} +applicationName: ${app_name} +appType: ${dockerBuildVersion} +deployment: + enabled: false + serviceAccount: + enabled: false +canary: + enabled: false +autoscaling: + enabled: false +podDisruptionBudget: + enabled: false + maxUnavailable: 100% + minAvailable: "" +cron: + enabled: true + concurrencyPolicy: ${concurrencyPolicy} + failedJobsHistoryLimit: ${failedJobsHistoryLimit} + successfulJobsHistoryLimit: ${successfulJobsHistoryLimit} + suspend: false + startingDeadlineSeconds: ${startingDeadlineSeconds} + backoffLimit: ${backoffLimit} + restartPolicy: ${restartPolicy} + env: + - name: PRISMSDK_ENVIRONMENT + value: ${prismsdk_environment} + envFrom: + secretRef: ${app_name} + image: + pullPolicy: IfNotPresent + pullSecret: "" + repository: ${registry}/${environment}/${build_team}/<% print module=='module_less'?repo_name.toLowerCase():repo_name.toLowerCase()+'/'+module+'' %> + tag: ${tag} + podAnnotations: + <% if (appMetrics && dockerBuildVersion.contains("maven")) { print 'jmx.io/path: /metrics' %> + <% print 'jmx.io/port: "8880"' %> + <% print 'jmx.io/scrape: "true"'} %> + <% if (nodeSelector.contains("arm64") && (environment=="int" || environment=="prd")) { print 'telegraf.influxdata.com/image: 847438129436.dkr.ecr.ap-southeast-1.amazonaws.com/telegraf:1.24.4-arm64' } %> + <% if ( CLOUD_PROVIDER == "GCP" ) { print 'telegraf.influxdata.com/image: asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/sre/telegraf:1.24.4' } %> + <% if (appMetrics) { print 'prometheus.io/path: /actuator/prometheus' %> + <% print 'prometheus.io/port: "'+app_port+'"' %> + <% print 'prometheus.io/scrape: "true"'} %> +<% if (pod_annotations) {pod_annotations.each{k,v -> if(v instanceof String) { println " ${k}: '${v}'"} else { println " ${k}: ${v}" }};} else {print ''} %> +<% if(serviceAccount){println ' serviceAccount:\n annotations:';serviceAccount.annotations.each{k,v -> println " ${k}: ${v}"};println " enabled: ${serviceAccount.enabled}"} else {print ' serviceAccount:\n annotations: null\n enabled: false'} %> +<% if (hostAliases){println ' hostAliases:';for(arr in hostAliases){println ' - ip: '+arr.get("ip"); println ' hostnames:'; for (ele in arr.get("hostnames")) { println ' - '+ele } }} %> + nodeSelector: + ${nodeSelector}: ${nodeSelectorValue} + tolerations: + - effect: NoSchedule + key: ${nodeSelector} + operator: Equal + value: ${nodeSelectorValue} +<% println ' jobs:'; for(val in jobs){ val.each{k,v -> println " ${k}:"; println " resources:"; println " limits:"; println " cpu: ${v.cpu_limit}"; println " memory: ${v.memory_limit}i"; println " requests:"; println " cpu: ${v.cpu_request}"; println " memory: ${v.memory_request}i"; println " schedule: \"${v.schedule}\""; if (v.command instanceof List){ println ' command:'; for (com in v.command ) { if (com != v.command.last()){ println " - ${com}" } else {println " - ${com};touch /tmp/podtermination/telegraf-termination"} }}; if(v.args){ if (v.args instanceof List){ println ' args:'; for (arg in v.args ) { println " - ${arg}" } }}}} %> +externalSecret: + annotations: +<% if (external_secrets_annotations) {external_secrets_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true + path: meesho/${vault_env}/${bu}/${team}/${app_name} + version: ${tag} + name: ${env_ns}-${app_name} + target: ${app_name} +ingress: + enabled: false +jmxconfig: + enabled: true +labels: + priority: <% print priority?:'p1' %> + priority_v2: <% print priority_v2?:'cp3' %> + primary_owner: ${primary_owner} + secondary_owner: ${secondary_owner} + env: ${environment_norm} + team: ${team_norm} + bu: ${bu_norm} + <% if (service_type_norm) { println "service_type: ${service_type_norm}" } %> + commit_id: ${commit_id} +nameOverride: "" +namespace: ${env_ns}-${app_name} +podSecurityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsUser: 65534 +service: + enabled: false + + + +appConfig: + enabled: <% if(appConfigEnabled) { print "true" } else { print "false"} %> + env: ${environment} +<% if(appConfigEnabled) {%> + staticAppConfig: + data: | +${staticAppConfigData.trim().replaceAll("(?m)^", " ")} + dynamicAppConfig: + data: | +${dynamicAppConfigData.trim().replaceAll("(?m)^", " ")} +<% } +%> diff --git a/resources/com/meesho/deployJar.yaml b/resources/com/meesho/deployJar.yaml new file mode 100644 index 0000000..2a3c34d --- /dev/null +++ b/resources/com/meesho/deployJar.yaml @@ -0,0 +1,203 @@ +--- +- name: Playbook to deploy Jar + hosts: all + gather_facts: false + become: yes + become_method: sudo + + tasks: + - name: Check if region is configured + shell: "aws configure get default.region" + register: aws_cli_region_reponse + + - name: "ERROR: AWS region missing" + vars: + msg: | + ############################################################################################################################# + ####### AWS REGION IS NOT CONFIGURED PROPERLY. CURRENT REGION: {{ aws_cli_region_reponse.stdout | upper }}, SHOULD BE: AP-SOUTHEAST-1 ####### + ############################################################################################################################# + debug: + msg: "{{ msg.split('\n') }}" + when: aws_cli_region_reponse.stdout != "ap-southeast-1" + failed_when: + - aws_cli_region_reponse.stdout != "ap-southeast-1" + + - name: Check if CICD profile is configured + shell: "aws configure get region --profile cicd" + register: aws_profile_cli_region_reponse + + - name: "ERROR: AWS region missing in profile" + vars: + msg: | + ############################################################################################################################# + ####### AWS REGION IS NOT CONFIGURED PROPERLY. CURRENT REGION: {{ aws_cli_region_reponse.stdout | upper }}, SHOULD BE: AP-SOUTHEAST-1 ####### + ############################################################################################################################# + debug: + msg: "{{ msg.split('\n') }}" + when: aws_profile_cli_region_reponse.stdout != "ap-southeast-1" + failed_when: + - aws_profile_cli_region_reponse.stdout != "ap-southeast-1" + + - name: Check if IAM role is attached + shell: "aws sts get-caller-identity" + register: aws_cli_gci_reponse + + - name: "ERROR: IAM role missing" + vars: + msg: | + ############################################################################################################################# + ############################################### CICD ROLE IS NOT ATTACHED ############################################### + ############################################################################################################################# + debug: + msg: "{{ msg.split('\n') }}" + when: + - aws_cli_gci_reponse.stdout is not search("assumed-role/cicd") + failed_when: + - aws_cli_gci_reponse.stdout is not search("assumed-role/cicd") + + - name: Check if systemd service exists + stat: + path: "/etc/systemd/system/{{ app_name }}.service" + register: service_status + + - name: Systemd service status + vars: + msg: | + ############################################################################################################################# + ################################ SYSTEMD SERVICE {{ app_name | upper }} DOES NOT EXIST. EXITING. ################################## + ############################################################################################################################# + debug: + msg: "{{ msg.split('\n') }}" + when: service_status is defined and not service_status.stat.exists + failed_when: + - not service_status.stat.exists + + - name: Check application directory structure + block: + + - name: Check application directory structure + stat: + path: "/home/ubuntu/{{ app_name }}" + register: app_dir_status + + - name: Application directory does not exist + vars: + msg: | + ############################################################################################################################# + ###################### APPLICATION DIRECTORY: /home/ubuntu/{{ app_name }} DOES NOT EXIST. EXITING. ####################### + ############################################################################################################################# + debug: + msg: "{{ msg.split('\n') }}" + when: app_dir_status is defined and not app_dir_status.stat.exists + failed_when: + - not app_dir_status.stat.exists + + tags: + - artifact_deployment + + - name: Ensure jq is installed + apt: + name: "jq" + state: present + force_apt_get: yes + + - name: Checking if environment file exists, pre-deployment + stat: + path: "/etc/sysconfig/{{ app_name }}" + register: env_file + + - name: Backing up current properties + copy: + src: "/etc/sysconfig/{{ app_name }}" + dest: "/etc/sysconfig/rollback-{{ app_name }}" + owner: ubuntu + group: ubuntu + mode: '0664' + remote_src: yes + force: yes + when: env_file.stat.exists + + - name: Checking if current symlink exists, pre-deployment + stat: + path: "/home/ubuntu/{{ app_name }}/{{ app_name }}-current.jar" + register: current_symlink + + - name: Show output + debug: msg= "{{ current_symlink }}" + + - name: Checking if current artifact exists, pre-deployment + stat: + path: "{{ current_symlink.stat.lnk_source }}" + when: + - current_symlink.stat.exists + - current_symlink.stat.islnk + register: current_artifact + + - name: Show output + debug: msg= "{{ current_artifact }}" + + - name: Stopping systemd service + systemd: + name: '{{ app_name }}' + state: stopped + + - name: Download latest properties + shell: "pull-env '{{ env }}' '{{ app_name }}'" + ignore_errors: yes + + - name: Download latest artifact + copy: + src: "{{ item }}" + dest: "/home/ubuntu/{{ app_name }}/" + owner: ubuntu + group: ubuntu + with_fileglob: + - "{{ repo_name }}/target/*.jar" + + - name: Setting latest_artifact_name + set_fact: + latest_artifact_name: '{{ item.split("/")[-1] }}' + with_fileglob: + - "{{ repo_name }}/target/*.jar" + + - name: Creating rollback symlink + file: + src: "{{ current_artifact.stat.path }}" + dest: "/home/ubuntu/{{ app_name }}/rollback" + state: link + + - name: Removing current symlink + file: + path: "/home/ubuntu/{{ app_name }}/{{ app_name }}-current.jar" + state: absent + + - name: Creating current symlink to latest artifact + file: + src: "/home/ubuntu/{{ app_name }}/{{ latest_artifact_name }}" + dest: "/home/ubuntu/{{ app_name }}/{{ app_name }}-current.jar" + state: link + + - name: Starting systemd service + systemd: + name: '{{ app_name }}' + state: started + daemon_reload: yes + + - name: Trying healthcheck + uri: + url: "http://localhost:{{ app_port }}{{ healthcheck_api }}" + method: GET + register: healthcheck_response + until: healthcheck_response.status == 200 + retries: 60 + delay: 1 + ignore_errors: False + + - name: Healthcheck response + vars: + msg: | + ############################################################################################################################## + HEALTH-CHECK ({{ healthcheck_api }}) RESPONSE: {{ healthcheck_response.status }} + ############################################################################################################################## + debug: + msg: "{{ msg.split('\n') }}" \ No newline at end of file diff --git a/resources/com/meesho/deployment.yaml b/resources/com/meesho/deployment.yaml new file mode 100644 index 0000000..4713e88 --- /dev/null +++ b/resources/com/meesho/deployment.yaml @@ -0,0 +1,103 @@ +--- + +app_name: ${app_name} +app_port: ${app_port} +health_check: ${health_check} +module: module_less +bu: ${bu} +team: ${team} +priority: ${priority} +priority_v2: ${priority_v2} +primary_owner: ${primary_owner} +secondary_owner: ${secondary_owner} +initialDelaySeconds: ${initialDelaySeconds} +replica_count: ${replica_count} +environment: + ftr: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - -Dspring.profiles.active=dev + - -XX:+UseG1GC + - -XX:+PrintGCDateStamps + - -XX:+PrintGCDetails + - -XX:+PrintGCApplicationStoppedTime + - -XX:+PrintGCApplicationConcurrentTime + - -XX:+PrintHeapAtGC + - -Xloggc:/var/log/gc.log + - -XX:+UseGCLogFileRotation + - -XX:NumberOfGCLogFiles=5 + - -XX:GCLogFileSize=9000k + stg: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - -Dspring.profiles.active=dev + - -XX:+UseG1GC + - -XX:+PrintGCDateStamps + - -XX:+PrintGCDetails + - -XX:+PrintGCApplicationStoppedTime + - -XX:+PrintGCApplicationConcurrentTime + - -XX:+PrintHeapAtGC + - -Xloggc:/var/log/gc.log + - -XX:+UseGCLogFileRotation + - -XX:NumberOfGCLogFiles=5 + - -XX:GCLogFileSize=9000k + int: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - -Dspring.profiles.active=int + - -XX:+UseG1GC + - -XX:+PrintGCDateStamps + - -XX:+PrintGCDetails + - -XX:+PrintGCApplicationStoppedTime + - -XX:+PrintGCApplicationConcurrentTime + - -XX:+PrintHeapAtGC + - -Xloggc:/var/log/gc.log + - -XX:+UseGCLogFileRotation + - -XX:NumberOfGCLogFiles=5 + - -XX:GCLogFileSize=9000k + prd: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - -Dspring.profiles.active=prd + - -XX:+UseG1GC + - -XX:+PrintGCDateStamps + - -XX:+PrintGCDetails + - -XX:+PrintGCApplicationStoppedTime + - -XX:+PrintGCApplicationConcurrentTime + - -XX:+PrintHeapAtGC + - -Xloggc:/var/log/gc.log + - -XX:+UseGCLogFileRotation + - -XX:NumberOfGCLogFiles=5 + - -XX:GCLogFileSize=9000k diff --git a/resources/com/meesho/go-Dockerfile b/resources/com/meesho/go-Dockerfile new file mode 100644 index 0000000..6942523 --- /dev/null +++ b/resources/com/meesho/go-Dockerfile @@ -0,0 +1,47 @@ +# tools and packages from base image +FROM ${build_registry}/build/golang:${version}-alpine3.18${kafka} AS builder + +<% if(module_property.kafka) { println "ENV GOPROXY=${go_proxy},direct CGO_ENABLED=1 GOOS=linux"} else { println "ENV GOPROXY=${go_proxy},direct CGO_ENABLED=0 GOOS=linux"} %> + +# Following steps will help in downloading dependencies for the project +COPY id_github_jenkins /root/.ssh/id_rsa + +RUN chmod 600 /root/.ssh/id_rsa && \ +ssh-keyscan github.com >> /root/.ssh/known_hosts && \ +git config --global url."git@github.com:".insteadOf "https://github.com/" + +# The tar file contains only the go.mod and go.sum files, it is generated as part of Jenkins build. +ADD only-mods.tar /app/ + +# directory inside container and default destination for all subsequent commands +<% if (base_dir) { println "WORKDIR /app/${base_dir}"} else { println "WORKDIR /app/"} %> + +# downloading all the dependencies +RUN --mount=type=cache,target=/go/pkg/mod,id=${repo_name} \ + --mount=type=cache,target=/root/.cache/go-build,id=${repo_name}-build \ + go env -w GOPRIVATE=github.com/Meesho/* && go mod download + +# copying the source code +COPY . /app/ + +# building the code for linux OS +RUN --mount=type=cache,target=/go/pkg/mod,id=${repo_name} \ + --mount=type=cache,target=/root/.cache/go-build,id=${repo_name}-build \ + go mod tidy && go build -tags musl --ldflags "-extldflags -static" -v -o /app/server cmd/${module}/main.go + +# deploy multistage +FROM ${build_registry}/build/debian:10-slim-v1 + +# install an up-to-date, properly rehashed CA trust store in the runtime image +# instead of copying the stale Alpine builder bundle (DEVOPS-30300) +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && update-ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# copy the binary to the production image from the builder stage. +COPY --from=builder /app/server /app/server +<% if (copy_file) { print "COPY copied_files/* $copy_target" } %> +<% add_files.each { print "COPY ${it.path} ${it.target}\n" } %> +# running the binary +CMD ["/app/server"] diff --git a/resources/com/meesho/go-deployment.yaml b/resources/com/meesho/go-deployment.yaml new file mode 100644 index 0000000..07d7688 --- /dev/null +++ b/resources/com/meesho/go-deployment.yaml @@ -0,0 +1,65 @@ +--- + +app_name: ${app_name} +app_port: ${app_port} +health_check: ${health_check} +module: module_less +bu: ${bu} +team: ${team} +priority: ${priority} +priority_v2: ${priority_v2} +primary_owner: ${primary_owner} +secondary_owner: ${secondary_owner} +initialDelaySeconds: ${initialDelaySeconds} +replica_count: ${replica_count} +environment: + ftr: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: false + command: /app/server + + dev: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: false + command: /app/server + int: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: false + command: /app/server + prd: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: false + command: /app/server + \ No newline at end of file diff --git a/resources/com/meesho/go-values.yaml b/resources/com/meesho/go-values.yaml new file mode 100644 index 0000000..d77700c --- /dev/null +++ b/resources/com/meesho/go-values.yaml @@ -0,0 +1,202 @@ +<% if (deploymentStrategy == 'canary'){ print "canary:\n enabled: true\n slackChannel: ${canary.slackChannel}\n enableManualPromotion: ${enableManualPromotion}\n skipAnalysis: ${canary.skipAnalysis}\n service:\n port: 80\n targetPort: ${(grpc_host || grpc_hosts) ? app_port : primary_port}\n progressDeadlineSeconds: ${canary.progressDeadlineSeconds}\n minCanaryReplicas: ${minCanaryReplicas}\n maxCanaryReplicas: ${maxCanaryReplicas}\n analysisInterval: ${canary.analysisInterval}\n analysisThreshold: ${canary.analysisThreshold}\n analysisMaxWeight: ${canary.analysisMaxWeight}\n analysisStepWeight: ${canary.analysisStepWeight}\n analysisMetrics:\n thresholdRangeMin: ${canary.analysisMetrics.thresholdRangeMin}\n interval: ${canary.analysisMetrics.interval}\n" } else { print "canary:\n enabled: false\n" } %> +repoName: ${repo_name} +cron: + enabled: false + serviceAccount: + enabled: false +applicationName: ${app_name} +autoscaling: + enabled: ${as_enabled} + maxReplicas: ${as_max} + minReplicas: ${as_min} + pollingInterval: ${as_poll} + scaledown: + policies: + - periodseconds: ${as_down_period} + type: Pods + value: ${as_down_pod_count} + selectpolicy: Min + stabilizationWindowSeconds: ${as_down_stable_window} + scaleup: + policies: + - periodseconds: ${as_up_period} + type: Pods + value: ${as_up_pod_count} + - periodseconds: ${as_up_period} + type: Percent + value: ${as_up_pod_percentage} + selectpolicy: Max + stabilizationWindowSeconds: ${as_up_stable_window} +<% if(!triggers){print " triggers:\n - metadata:\n value: \"${as_trigger_value}\"\n metricType: ${as_trigger_type}\n type: ${as_trigger_metric}"} else {println ' triggers:'; for(val in triggers){if(val instanceof Map){ val.each{k,v -> if (v instanceof Map) { println ' - '+k+':'; v.each{a,b -> if(b.isNumber()){println " ${a}: \"${b}\"" } else {println " ${a}: ${b}" }}} else {println" ${k}: ${v}"}}}}} %> +replicaCount: ${replica_count} +deployment: + affinity: {} + podDistributionSkew: ${podDistributionSkew} +<% if (deployment_args){println ' args:'; for (arg in deployment_args){ println ' - '+arg}} %> + command: + - ${command} + enabled: true + env: + - name: GOMAXPROCS + value: ${activeProcessorCount} + envFrom: + secretRef: ${app_name} + image: + pullPolicy: IfNotPresent + pullSecret: "" + repository: ${registry}/${environment}/${build_team}/<% print module=='module_less'?repo_name.toLowerCase():repo_name.toLowerCase()+'/'+module %> + tag: ${tag} +<% if(lifecycle) { println ' lifecycle:\n preStop:\n exec:\n command:'; for(val in lifecycle.preStop.exec.command){ println " - $val" }} else {print " lifecycle:\n preStop:\n exec:\n command:\n - /bin/bash\n - -c\n - kill -SIGQUIT 1 ; /bin/sleep 120\n "} %> + minReadySeconds: 10 + podAnnotations: + <% if (appMetrics) { print 'prometheus.io/path: /actuator/prometheus' %> + <% print 'prometheus.io/port: "'+app_port+'"' %> + <% print 'prometheus.io/scrape: "true"'} %> + <% if (nodeSelector.contains("arm64") && (environment=="int" || environment=="prd")) { print 'telegraf.influxdata.com/image: 847438129436.dkr.ecr.ap-southeast-1.amazonaws.com/telegraf:1.24.4-arm64' } %> +<% if (pod_annotations) {pod_annotations.each{k,v -> println " ${k}: '${v}'"};} else {print ''} %> + ports: + - containerPort: ${app_port} + name: http + protocol: TCP + - containerPort: 8880 + name: metric + protocol: TCP + probes: + liveness: + failureThreshold: ${liveness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${liveness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${liveness_success_threshold} + timeoutSeconds: ${liveness_timeout_seconds} + readiness: + failureThreshold: ${readiness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${readiness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${readiness_success_threshold} + timeoutSeconds: ${readiness_timeout_seconds} + resources: + limits: + cpu: ${cpu_limit} + memory: ${memory_limit}i + requests: + cpu: ${cpu_request} + memory: ${memory_request}i + revisionHistoryLimit: 6 +<% if(serviceAccount){println ' serviceAccount:\n annotations:';serviceAccount.annotations.each{k,v -> println " ${k}: ${v}"};println " enabled: ${serviceAccount.enabled}"} else {print ' serviceAccount:\n annotations: null\n enabled: false'} %> + nodeSelector: + ${nodeSelector}: ${nodeSelectorValue} +<% if (hostAliases){println ' hostAliases:';for(arr in hostAliases){println ' - ip: '+arr.get("ip"); println ' hostnames:'; for (ele in arr.get("hostnames")) { println ' - '+ele } }} %> + tolerations: + - effect: NoSchedule + key: ${nodeSelector} + operator: Equal + value: ${nodeSelectorValue} + updateStrategy: + strategy: +<% if (deploymentStrategy == 'recreate') { print " type: Recreate"} else { print " type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 0%\n maxSurge: ${maxSurge}%" } %> +externalSecret: + annotations: +<% if (external_secrets_annotations) {external_secrets_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true + path: meesho/${vault_env}/${bu}/${team}/${app_name} + version: ${tag} + name: ${env_ns}-${app_name} + target: ${app_name} +fullnameOverride: "" +ingress: + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/server-snippet: | + location ~* "^/api/1.0/search/recent" { + default_type application/json; + return 200 '{"recent_searches": [],"recent_suggestions": [],"limit": 5,"autosuggest_use_recent": true,"show_recent_header": false}'; + } + location ~* "^/api/1.0/anonymous/search/recent" { + default_type application/json; + return 200 '{"recent_searches": [],"recent_suggestions": [],"limit": 5,"autosuggest_use_recent": true,"show_recent_header": false}'; + } + location ~* "^/search-queries/recent" { + default_type application/json; + return 200 '{"recent_searches": [],"recent_suggestions": [],"limit": 5,"autosuggest_use_recent": true,"show_recent_header": false}'; + } +<% if (ingress_annotations) {ingress_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true +<% if(grpc_host){println " grpc_hosts:\n - host: ${grpc_host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else if (grpc_hosts) {println " grpc_hosts:";for(host_arr in grpc_hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> +<% if(host){println " hosts:\n - host: ${host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else {println " hosts:";for(host_arr in hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> ingressClassName: ${ingress_class} + servicePort: http + enableWebsocket: ${enableWebsocket} + slowStart: + enabled: <% if (slowStartWindow) { print "true" } else { print "false" } %> + window: <% if (slowStartWindow) { print "${slowStartWindow}" } else { print "120s" } %> + aggression: <% if (slowStartAggression) { print "${slowStartAggression}" } else { print "1.0" } %> + minPercent: <% if (slowStartMinPercent) { print "${slowStartMinPercent}" } else { print "10" } %> +jmxconfig: + enabled: false +labels: + priority: <% print priority?:'p1' %> + priority_v2: <% print priority_v2?:'cp3' %> + primary_owner: ${primary_owner} + secondary_owner: ${secondary_owner} + env: ${environment_norm} + team: ${team_norm} + bu: ${bu_norm} + <% if (service_type_norm) { println "service_type: ${service_type_norm}" } %> + commit_id: ${commit_id} +nameOverride: "" +namespace: ${env_ns}-${app_name} +podDisruptionBudget: + enabled: <% if (pdbMaxUnavailable) { print "true" } else { print "false" } %> + maxUnavailable: ${pdbMaxUnavailable} + minAvailable: ${pdbMinAvailable} +podSecurityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsUser: 65534 +service: +<% if (service_annotations) { println " annotations:"; service_annotations.each{k,v -> println " ${k}: \"${v}\""};} else { println " annotations: null" } %> + enabled: true +<% if (addon_ports) { for (p in addon_ports) { println " addons:"; println " - name: ${p.name}"; println " targetPort: ${p.targetPort}"; println " type: ${p.type}"; }} else { println " addon_ports: []" } %> +<% if (grpc_port && app_port && (grpc_host || grpc_hosts)) { + println " ports:" + println " - name: http" + println " port: 80" + println " protocol: TCP" + println " targetPort: ${app_port}" + println " grpc_ports:" + println " - name: grpc" + println " port: 80" + println " protocol: TCP" + println " targetPort: ${grpc_port}" +} else { + println " ports:" + println " - name: http" + println " port: 80" + println " protocol: TCP" + println " targetPort: ${primary_port}" +} +%> + type: ClusterIP + +createContourGateway: <% if (createContourGateway) { print "${createContourGateway}" } else { print "false" } %> +contourResponseTimeout: ${contourResponseTimeout} + +appConfig: + enabled: <% if(appConfigEnabled) { print "true" } else { print "false"} %> + env: ${environment} +<% if(appConfigEnabled) {%> + staticAppConfig: + data: | +${staticAppConfigData.trim().replaceAll("(?m)^", " ")} + dynamicAppConfig: + data: | +${dynamicAppConfigData.trim().replaceAll("(?m)^", " ")} +<% } +%> diff --git a/resources/com/meesho/gradle-deployment.yaml b/resources/com/meesho/gradle-deployment.yaml new file mode 100644 index 0000000..4713e88 --- /dev/null +++ b/resources/com/meesho/gradle-deployment.yaml @@ -0,0 +1,103 @@ +--- + +app_name: ${app_name} +app_port: ${app_port} +health_check: ${health_check} +module: module_less +bu: ${bu} +team: ${team} +priority: ${priority} +priority_v2: ${priority_v2} +primary_owner: ${primary_owner} +secondary_owner: ${secondary_owner} +initialDelaySeconds: ${initialDelaySeconds} +replica_count: ${replica_count} +environment: + ftr: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - -Dspring.profiles.active=dev + - -XX:+UseG1GC + - -XX:+PrintGCDateStamps + - -XX:+PrintGCDetails + - -XX:+PrintGCApplicationStoppedTime + - -XX:+PrintGCApplicationConcurrentTime + - -XX:+PrintHeapAtGC + - -Xloggc:/var/log/gc.log + - -XX:+UseGCLogFileRotation + - -XX:NumberOfGCLogFiles=5 + - -XX:GCLogFileSize=9000k + stg: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - -Dspring.profiles.active=dev + - -XX:+UseG1GC + - -XX:+PrintGCDateStamps + - -XX:+PrintGCDetails + - -XX:+PrintGCApplicationStoppedTime + - -XX:+PrintGCApplicationConcurrentTime + - -XX:+PrintHeapAtGC + - -Xloggc:/var/log/gc.log + - -XX:+UseGCLogFileRotation + - -XX:NumberOfGCLogFiles=5 + - -XX:GCLogFileSize=9000k + int: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - -Dspring.profiles.active=int + - -XX:+UseG1GC + - -XX:+PrintGCDateStamps + - -XX:+PrintGCDetails + - -XX:+PrintGCApplicationStoppedTime + - -XX:+PrintGCApplicationConcurrentTime + - -XX:+PrintHeapAtGC + - -Xloggc:/var/log/gc.log + - -XX:+UseGCLogFileRotation + - -XX:NumberOfGCLogFiles=5 + - -XX:GCLogFileSize=9000k + prd: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - -Dspring.profiles.active=prd + - -XX:+UseG1GC + - -XX:+PrintGCDateStamps + - -XX:+PrintGCDetails + - -XX:+PrintGCApplicationStoppedTime + - -XX:+PrintGCApplicationConcurrentTime + - -XX:+PrintHeapAtGC + - -Xloggc:/var/log/gc.log + - -XX:+UseGCLogFileRotation + - -XX:NumberOfGCLogFiles=5 + - -XX:GCLogFileSize=9000k diff --git a/resources/com/meesho/java-Dockerfile b/resources/com/meesho/java-Dockerfile new file mode 100644 index 0000000..bcd4726 --- /dev/null +++ b/resources/com/meesho/java-Dockerfile @@ -0,0 +1,33 @@ +# This sample, non-production-ready template describes an Amazon EC2 instance and an Elastic Load Balancer. +# © 2020 Amazon Web Services, Inc. or its affiliates. All Rights Reserved. +# This AWS Content is provided subject to the terms of the AWS Customer Agreement available at +# http://aws.amazon.com/agreement or other written agreement between Customer and either +# Amazon Web Services, Inc. or Amazon Web Services EMEA SARL or both. + +FROM ${buildRegistry}/build/java:${dockerBuildVersion} + +ARG artifactId=<% print module=='module_less'?repo_name:module %> +ARG XMS=2G +ARG XMX=2G + +ADD https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar /opt/jmx_exporter.jar +#ADD https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.17.0/opentelemetry-javaagent.jar /opt/opentelemetry-javaagent.jar + +ADD https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.26.0/opentelemetry-javaagent.jar /opt/opentelemetry-javaagent.jar + +EXPOSE 8880 8010 + +COPY <% print module=='module_less'?'':module+"/" %>target/*.jar /opt/target/<% print '$artifactId' %>.jar +#RUN mkdir -p /var/log/<% print '$artifactId' %> && touch /var/log/<% print '$artifactId' %>/gc.log +<% add_files.each { print "COPY ${it.path} ${it.target}\n" } %> +<% if (copy_file) { print "COPY copied_files/* $copy_target"} %> + +WORKDIR /opt/target + +CMD ["<% print '$artifactId' %>.jar", "-javaagent:/opt/jmx_exporter.jar=8880:/opt/config/jmx-config.yaml", \\ + "-XX:MinRAMPercentage=50.0", "-XX:MaxRAMPercentage=80.0", \\ + "-XX:+UseParallelGC -XX:+PrintGCDateStamps -XX:+PrintGCDetails", \\ + "-XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime", "-XX:+PrintHeapAtGC", \\ + "-Xloggc:/var/log/gc.log", \\ + "-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=9000k", \\ + "-Xms<% print '$XMS'%>", "-Xmx<% print '$XMX'%>"] diff --git a/resources/com/meesho/node-Dockerfile b/resources/com/meesho/node-Dockerfile new file mode 100644 index 0000000..4c1d08a --- /dev/null +++ b/resources/com/meesho/node-Dockerfile @@ -0,0 +1,58 @@ +FROM ${buildRegistry}/build/node:${version}-slim-secure-multiarch_v2.0 as build-env + +WORKDIR /usr/src/app + +COPY id_github_jenkins /root/.ssh/id_rsa + +RUN ssh-keyscan github.com >> /root/.ssh/known_hosts + +COPY . ./ +<% add_files.each { print "COPY ${it.path} ${it.target}\n" } %> + +RUN ls -al && \ + echo 'NPM Install...' && \ + <% if (npm_install_arg) { %>\ + echo 'Using custom npm install command: ${npm_install_arg}' && \ + ${npm_install_arg} && \ + <% } else { %>\ + npm ci && \ + <% } %>\ + echo 'Ensuring .npmrc is available in subdirectories...' && \ + for subdir in server client; do \ + if [ -d "\$subdir" ] && [ -f .npmrc ] && [ ! -f "\$subdir/.npmrc" ]; then \ + cp -v .npmrc "\$subdir/.npmrc"; \ + fi; \ + done && \ + <% if (pbac_enabled) { %>\ + echo 'Installing PBAC Registry Sync CLI...' && \ + git config --global url."git@github.com:".insteadOf "https://github.com/" && \ + npm install -g pbac-registry-sync-cli --registry=${npm_registry} && \ + echo 'Syncing PBAC Registry for scope: ${pbac_scope_name} environment: ${pbac_env}' && \ + sync-pbac-registry ${pbac_scope_name} ${pbac_env} && \ + echo 'PBAC Registry Sync Completed' && \ + echo 'Copying .npmrc to subdirectories for proper registry resolution...' && \ + for subdir in server client; do \ + if [ -d "\$subdir" ] && [ ! -f "\$subdir/.npmrc" ]; then \ + cp -v .npmrc "\$subdir/.npmrc"; \ + fi; \ + done && \ + <% } %>\ + if ! ${skip_sonar}; then \ + cat .env | grep 'SONAR_' && \ + echo 'Installing Java 17...' && \ + if [ "\$(cat /etc/os-release | grep 'ID=' | awk -F= '{print \$2}' | sed '2d')" = 'alpine' ]; then \ + echo "Alpine" && \ + apk add --no-cache openjdk17; \ + else \ + echo "Debian" && \ + apt-get update && \ + apt-get install -y --no-install-recommends openjdk-17-jre && \ + apt-get clean; \ + fi && \ + echo 'Performing Test Coverage...' && \ + npm run ${testCMD} || true && \ + echo 'Static Analysis...' && \ + npm run sonar -- source=${scmType}; \ + fi && \ + echo 'NPM build...' && \ + npm run build diff --git a/resources/com/meesho/node-deployment.yaml b/resources/com/meesho/node-deployment.yaml new file mode 100644 index 0000000..08ccde1 --- /dev/null +++ b/resources/com/meesho/node-deployment.yaml @@ -0,0 +1,64 @@ +--- + +app_name: ${app_name} +app_port: ${app_port} +health_check: ${health_check} +module: module_less +bu: ${bu} +team: ${team} +priority: ${priority} +priority_v2: ${priority_v2} +primary_owner: ${primary_owner} +secondary_owner: ${secondary_owner} +initialDelaySeconds: ${initialDelaySeconds} +replica_count: ${replica_count} +environment: + ftr: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - pm2-config.json + stg: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - pm2-config.json + int: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - pm2-config.json + prd: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + - pm2-config.json + \ No newline at end of file diff --git a/resources/com/meesho/node-values.yaml b/resources/com/meesho/node-values.yaml new file mode 100644 index 0000000..e5aec68 --- /dev/null +++ b/resources/com/meesho/node-values.yaml @@ -0,0 +1,184 @@ +<% if (deploymentStrategy == 'canary'){ print "canary:\n enabled: true\n slackChannel: ${canary.slackChannel}\n enableManualPromotion: ${enableManualPromotion}\n skipAnalysis: ${canary.skipAnalysis}\n service:\n port: 80\n targetPort: ${primary_port}\n progressDeadlineSeconds: ${canary.progressDeadlineSeconds}\n minCanaryReplicas: ${minCanaryReplicas}\n maxCanaryReplicas: ${maxCanaryReplicas}\n analysisInterval: ${canary.analysisInterval}\n analysisThreshold: ${canary.analysisThreshold}\n analysisMaxWeight: ${canary.analysisMaxWeight}\n analysisStepWeight: ${canary.analysisStepWeight}\n analysisMetrics:\n thresholdRangeMin: ${canary.analysisMetrics.thresholdRangeMin}\n interval: ${canary.analysisMetrics.interval}\n" } else { print "canary:\n enabled: false\n" } %>repoName: ${repo_name} +cron: + enabled: false + serviceAccount: + enabled: false +applicationName: ${app_name} +autoscaling: + enabled: ${as_enabled} + maxReplicas: ${as_max} + minReplicas: ${as_min} + pollingInterval: ${as_poll} + scaledown: + policies: + - periodseconds: ${as_down_period} + type: Pods + value: ${as_down_pod_count} + selectpolicy: Min + stabilizationWindowSeconds: ${as_down_stable_window} + scaleup: + policies: + - periodseconds: ${as_up_period} + type: Pods + value: ${as_up_pod_count} + - periodseconds: ${as_up_period} + type: Percent + value: ${as_up_pod_percentage} + selectpolicy: Max + stabilizationWindowSeconds: ${as_up_stable_window} +<% if(!triggers){print " triggers:\n - metadata:\n value: \"${as_trigger_value}\"\n metricType: ${as_trigger_type}\n type: ${as_trigger_metric}"} else {println ' triggers:'; for(val in triggers){if(val instanceof Map){ val.each{k,v -> if (v instanceof Map) { println ' - '+k+':'; v.each{a,b -> if(b.isNumber()){println " ${a}: \"${b}\"" } else {println " ${a}: ${b}" }}} else {println" ${k}: ${v}"}}}}} %> +deployment: + affinity: {} + podDistributionSkew: ${podDistributionSkew} + args: +<% for (arg in deployment_args){ println ' - '+arg}%> + command: + - ${command} + enabled: true + env: null + envFrom: + secretRef: ${app_name} + image: + pullPolicy: IfNotPresent + pullSecret: "" + repository: ${registry}/${environment}/${build_team}/<% print module=='module_less'?repo_name.toLowerCase():repo_name.toLowerCase()+'/'+module %> + tag: ${tag} + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - kill -SIGQUIT 1 ; /bin/sleep 60 + minReadySeconds: 10 + podAnnotations: + <% if (appMetrics) { print 'prometheus.io/path: "/actuator/prometheus"' %> + <% print 'prometheus.io/port: "'+app_port+'"' %> + <% print 'prometheus.io/scrape: "true"'} %> +<% if (pod_annotations) {pod_annotations.each{k,v -> println " ${k}: \"${v}\""};} else {print ''} %> + ports: + - containerPort: ${app_port} + name: http + protocol: TCP + - containerPort: 9209 + name: pm2-metrics + protocol: TCP + - containerPort: 9200 + name: metrics + protocol: TCP + probes: + liveness: + failureThreshold: ${liveness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${liveness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${liveness_success_threshold} + timeoutSeconds: ${liveness_timeout_seconds} + readiness: + failureThreshold: ${readiness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${readiness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${readiness_success_threshold} + timeoutSeconds: ${readiness_timeout_seconds} + replicaCount: ${replica_count} + resources: + limits: + cpu: ${cpu_limit} + memory: ${memory_limit}i + requests: + cpu: ${cpu_request} + memory: ${memory_request}i + revisionHistoryLimit: 6 +<% if(serviceAccount){println ' serviceAccount:\n annotations:';serviceAccount.annotations.each{k,v -> println " ${k}: ${v}"};println " enabled: ${serviceAccount.enabled}"} else {print ' serviceAccount:\n annotations: null\n enabled: false'} %> + nodeSelector: + ${nodeSelector}: ${nodeSelectorValue} +<% if (hostAliases){println ' hostAliases:';for(arr in hostAliases){println ' - ip: '+arr.get("ip"); println ' hostnames:'; for (ele in arr.get("hostnames")) { println ' - '+ele } }} %> + tolerations: + - effect: NoSchedule + key: ${nodeSelector} + operator: Equal + value: ${nodeSelectorValue} + updateStrategy: + strategy: +<% if (deploymentStrategy == 'recreate') { print " type: Recreate"} else { print " type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 0%\n maxSurge: ${maxSurge}%" } %> +externalSecret: + annotations: +<% if (external_secrets_annotations) {external_secrets_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: <% if(appConfigEnabled) { print "true" } else { print "false"} %> + path: meesho/${vault_env}/${bu}/${team}/${app_name} + version: ${tag} + name: ${env_ns}-${app_name} + target: ${app_name} +fullnameOverride: "" +ingress: + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "false" +<% if (ingress_annotations) {ingress_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true +<% if(grpc_host){println " grpc_hosts:\n - host: ${grpc_host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else if (grpc_hosts) {println " grpc_hosts:";for(host_arr in grpc_hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> +<% if(host){println " hosts:\n - host: ${host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else {println " hosts:";for(host_arr in hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> ingressClassName: ${ingress_class} + servicePort: http + enableWebsocket: ${enableWebsocket} + slowStart: + enabled: <% if (slowStartWindow) { print "true" } else { print "false" } %> + window: <% if (slowStartWindow) { print "${slowStartWindow}" } else { print "120s" } %> + aggression: <% if (slowStartAggression) { print "${slowStartAggression}" } else { print "1.0" } %> + minPercent: <% if (slowStartMinPercent) { print "${slowStartMinPercent}" } else { print "10" } %> +jmxconfig: + enabled: false +labels: + priority: <% print priority?:'p1' %> + priority_v2: <% print priority_v2?:'cp3' %> + primary_owner: ${primary_owner} + secondary_owner: ${secondary_owner} + env: ${environment_norm} + team: ${team_norm} + bu: ${bu_norm} + <% if (service_type_norm) { println "service_type: ${service_type_norm}" } %> + commit_id: ${commit_id} + node_prom_client_enabled: true +nameOverride: "" +replicaCount: ${replica_count} +namespace: ${env_ns}-${app_name} +podDisruptionBudget: + enabled: <% if (pdbMaxUnavailable) { print "true" } else { print "false" } %> + maxUnavailable: ${pdbMaxUnavailable} + minAvailable: ${pdbMinAvailable} +podSecurityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsUser: 65534 +service: +<% if (service_annotations) { println " annotations:"; service_annotations.each{k,v -> println " ${k}: \"${v}\""};} else { println " annotations: null" } %> + enabled: true +<% if (addon_ports) { for (p in addon_ports) { println " addons:"; println " - name: ${p.name}"; println " targetPort: ${p.targetPort}"; println " type: ${p.type}"; }} else { println " addon_ports: []" } %> + ports: + - name: http + port: 80 + protocol: TCP + targetPort: ${primary_port} + type: ClusterIP + +createContourGateway: <% if (createContourGateway) { print "${createContourGateway}" } else { print "false" } %> +contourResponseTimeout: ${contourResponseTimeout} + + +appConfig: + enabled: <% if(appConfigEnabled) { print "true" } else { print "false"} %> + env: ${environment} +<% if(appConfigEnabled) {%> + staticAppConfig: + data: | +${staticAppConfigData.trim().replaceAll("(?m)^", " ")} + dynamicAppConfig: + data: | +${dynamicAppConfigData.trim().replaceAll("(?m)^", " ")} +<% } +%> diff --git a/resources/com/meesho/php-Dockerfile b/resources/com/meesho/php-Dockerfile new file mode 100644 index 0000000..aa9784a --- /dev/null +++ b/resources/com/meesho/php-Dockerfile @@ -0,0 +1,35 @@ +FROM php:7.0.33-apache +RUN echo "deb http://archive.debian.org/debian stretch main" > /etc/apt/sources.list + +RUN apt-get update && \ + apt-get install -y libxml2-dev && \ + apt-get install -y libssl-dev && \ + apt-get install -y libedit-dev && \ + apt-get install -y libpq-dev && \ + apt-get install -y libxslt1-dev && \ + apt-get install -y zlib1g-dev && \ + apt-get install -y libxml2-dev + +RUN docker-php-ext-install xml xmlwriter mysqli pdo pdo_mysql opcache json calendar fileinfo ctype dom exif ftp gettext iconv pgsql phar posix readline shmop simplexml sockets sysvmsg sysvsem sysvshm tokenizer wddx xsl pdo_pgsql + +RUN mkdir -p /home/ubuntu/billion_dollar/supply_api +RUN mkdir -p /home/ubuntu/billion_dollar/supply_api/logs/ + +COPY ./ /home/ubuntu/billion_dollar/supply_api +<% if (copy_file) { print "COPY copied_files/apache2/ ${copy_target}\nRUN mkdir -p /etc/phpmyadmin && mv ${copy_target}phpmyadmin_apache.conf /etc/phpmyadmin/apache.conf"} %> +<% add_files.each { print "COPY ${it.path} ${it.target}\n" } %> + +WORKDIR /home/ubuntu/billion_dollar/supply_api +RUN cp /usr/local/etc/php/php.ini-production /usr/local/etc/php/php.ini +RUN echo "date.timezone=Asia/Kolkata" >> /usr/local/etc/php/php.ini + +RUN chmod 755 /home/ubuntu/billion_dollar/supply_api + +RUN service apache2 restart + +RUN rm /etc/apache2/sites-enabled/000-default.conf && \ + rm /etc/apache2/sites-available/000-default.conf && \ + chmod -R 755 /home/ubuntu/billion_dollar/supply_api/api && \ + chmod 644 /home/ubuntu/billion_dollar/supply_api/api/.htaccess + +CMD ["apache2-foreground"] diff --git a/resources/com/meesho/php-deployment.yaml b/resources/com/meesho/php-deployment.yaml new file mode 100644 index 0000000..0347f9e --- /dev/null +++ b/resources/com/meesho/php-deployment.yaml @@ -0,0 +1,55 @@ +--- + +app_name: ${app_name} +app_port: ${app_port} +health_check: ${health_check} +module: module_less +bu: ${bu} +team: ${team} +priority: ${priority} +priority_v2: ${priority_v2} +primary_owner: ${primary_owner} +secondary_owner: ${secondary_owner} +initialDelaySeconds: ${initialDelaySeconds} +replica_count: ${replica_count} +environment: + ftr: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + stg: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + int: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + prd: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} \ No newline at end of file diff --git a/resources/com/meesho/php-values.yaml b/resources/com/meesho/php-values.yaml new file mode 100644 index 0000000..cf1f91c --- /dev/null +++ b/resources/com/meesho/php-values.yaml @@ -0,0 +1,169 @@ +<% if (deploymentStrategy == 'canary'){ print "canary:\n enabled: true\n slackChannel: ${canary.slackChannel}\n enableManualPromotion: ${enableManualPromotion}\n skipAnalysis: ${canary.skipAnalysis}\n service:\n port: 80\n targetPort: ${primary_port}\n progressDeadlineSeconds: ${canary.progressDeadlineSeconds}\n minCanaryReplicas: ${minCanaryReplicas}\n maxCanaryReplicas: ${maxCanaryReplicas}\n analysisInterval: ${canary.analysisInterval}\n analysisThreshold: ${canary.analysisThreshold}\n analysisMaxWeight: ${canary.analysisMaxWeight}\n analysisStepWeight: ${canary.analysisStepWeight}\n analysisMetrics:\n thresholdRangeMin: ${canary.analysisMetrics.thresholdRangeMin}\n interval: ${canary.analysisMetrics.interval}\n" } else { print "canary:\n enabled: false\n" } %>repoName: ${repo_name} +cron: + enabled: false + serviceAccount: + enabled: false +applicationName: ${app_name} +appType: php +autoscaling: + enabled: ${as_enabled} + maxReplicas: ${as_max} + minReplicas: ${as_min} + pollingInterval: ${as_poll} + scaledown: + policies: + - periodseconds: ${as_down_period} + type: Pods + value: ${as_down_pod_count} + selectpolicy: Min + stabilizationWindowSeconds: 300 + scaleup: + policies: + - periodseconds: ${as_up_period} + type: Pods + value: ${as_up_pod_count} + - periodseconds: ${as_up_period} + type: Percent + value: ${as_up_pod_percentage} + selectpolicy: Max + stabilizationWindowSeconds: ${as_up_stable_window} +<% if(!triggers){print " triggers:\n - metadata:\n value: \"${as_trigger_value}\"\n metricType: ${as_trigger_type}\n type: ${as_trigger_metric}"} else {println ' triggers:'; for(val in triggers){if(val instanceof Map){ val.each{k,v -> if (v instanceof Map) { println ' - '+k+':'; v.each{a,b -> if(b.isNumber()){println " ${a}: \"${b}\"" } else {println " ${a}: ${b}" }}} else {println" ${k}: ${v}"}}}}} %> +deployment: + affinity: {} + podDistributionSkew: ${podDistributionSkew} + command: + - apache2-foreground + enabled: true + env: null + envFrom: + secretRef: ${app_name} + image: + pullPolicy: IfNotPresent + pullSecret: "" + repository: ${registry}/${environment}/${build_team}/<% print module=='module_less'?repo_name.toLowerCase():repo_name.toLowerCase()+'/'+module %> + tag: ${tag} +<% if(lifecycle) { println ' lifecycle:\n preStop:\n exec:\n command:'; for(val in lifecycle.preStop.exec.command){ println " - $val" }} else {print " lifecycle:\n preStop:\n exec:\n command:\n - /bin/bash\n - -c\n - kill -SIGQUIT 1 ; /bin/sleep 120\n "} %> + minReadySeconds: 10 + podAnnotations: + <% if (nodeSelector.contains("arm64") && (environment=="int" || environment=="prd")) { print 'telegraf.influxdata.com/image: 847438129436.dkr.ecr.ap-southeast-1.amazonaws.com/telegraf:1.24.4-arm64' } %> + <% if (appMetrics) { print 'prometheus.io/path: /actuator/prometheus' %> + <% print 'prometheus.io/port: "'+app_port+'"' %> + <% print 'prometheus.io/scrape: "true"'} %> +<% if (pod_annotations) {pod_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + ports: + - containerPort: ${app_port} + name: http + protocol: TCP + probes: + liveness: + failureThreshold: ${liveness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${liveness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${liveness_success_threshold} + timeoutSeconds: ${liveness_timeout_seconds} + readiness: + failureThreshold: ${readiness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${readiness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${readiness_success_threshold} + timeoutSeconds: ${readiness_timeout_seconds} + replicaCount: ${replica_count} + resources: + limits: + cpu: ${cpu_limit} + memory: ${memory_limit}i + requests: + cpu: ${cpu_request} + memory: ${memory_request}i + revisionHistoryLimit: 6 +<% if(serviceAccount){println ' serviceAccount:\n annotations:';serviceAccount.annotations.each{k,v -> println " ${k}: ${v}"};println " enabled: ${serviceAccount.enabled}"} else {print ' serviceAccount:\n annotations: null\n enabled: false'} %> + nodeSelector: + ${nodeSelector}: ${nodeSelectorValue} +<% if (hostAliases){println ' hostAliases:';for(arr in hostAliases){println ' - ip: '+arr.get("ip"); println ' hostnames:'; for (ele in arr.get("hostnames")) { println ' - '+ele } }} %> + tolerations: + - effect: NoSchedule + key: ${nodeSelector} + operator: Equal + value: ${nodeSelectorValue} + updateStrategy: + strategy: +<% if (deploymentStrategy == 'recreate') { print " type: Recreate"} else { print " type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 0%\n maxSurge: ${maxSurge}%" } %> +externalSecret: + annotations: +<% if (external_secrets_annotations) {external_secrets_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true + path: meesho/${vault_env}/${bu}/${team}/${app_name} + version: ${tag} + name: ${env_ns}-${app_name} + target: ${app_name} +fullnameOverride: "" +replicaCount: ${replica_count} +ingress: + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" + nginx.ingress.kubernetes.io/ssl-redirect: "false" +<% if (ingress_annotations) {ingress_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true +<% if(grpc_host){println " grpc_hosts:\n - host: ${grpc_host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else if (grpc_hosts) {println " grpc_hosts:";for(host_arr in grpc_hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> +<% if(host){println " hosts:\n - host: ${host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else {println " hosts:";for(host_arr in hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> ingressClassName: ${ingress_class} + servicePort: http + enableWebsocket: ${enableWebsocket} + slowStart: + enabled: <% if (slowStartWindow) { print "true" } else { print "false" } %> + window: <% if (slowStartWindow) { print "${slowStartWindow}" } else { print "120s" } %> + aggression: <% if (slowStartAggression) { print "${slowStartAggression}" } else { print "1.0" } %> + minPercent: <% if (slowStartMinPercent) { print "${slowStartMinPercent}" } else { print "10" } %> +jmxconfig: + enabled: false +labels: + priority: <% print priority?:'p1' %> + priority_v2: <% print priority_v2?:'cp3' %> + primary_owner: ${primary_owner} + secondary_owner: ${secondary_owner} + env: ${environment_norm} + team: ${team_norm} + bu: ${bu_norm} + <% if (service_type_norm) { println "service_type: ${service_type_norm}" } %> + commit_id: ${commit_id} +nameOverride: "" +namespace: ${env_ns}-${app_name} +podDisruptionBudget: + enabled: <% if (pdbMaxUnavailable) { print "true" } else { print "false" } %> + maxUnavailable: ${pdbMaxUnavailable} + minAvailable: ${pdbMinAvailable} +podSecurityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsUser: 65534 +service: +<% if (service_annotations) { println " annotations:"; service_annotations.each{k,v -> println " ${k}: \"${v}\""};} else { println " annotations: null" } %> + enabled: true +<% if (addon_ports) { for (p in addon_ports) { println " addons:"; println " - name: ${p.name}"; println " targetPort: ${p.targetPort}"; println " type: ${p.type}"; }} else { println " addon_ports: []" } %> + ports: + - name: http + port: 80 + protocol: TCP + targetPort: ${primary_port} + type: ClusterIP + +createContourGateway: <% if (createContourGateway) { print "${createContourGateway}" } else { print "false" } %> +contourResponseTimeout: ${contourResponseTimeout} + +appConfig: + enabled: <% if(appConfigEnabled) { print "true" } else { print "false"} %> + env: ${environment} +<% if(appConfigEnabled) {%> + staticAppConfig: + data: | +${staticAppConfigData.trim().replaceAll("(?m)^", " ")} + dynamicAppConfig: + data: | +${dynamicAppConfigData.trim().replaceAll("(?m)^", " ")} +<% } +%> \ No newline at end of file diff --git a/resources/com/meesho/python-2.7-Dockerfile b/resources/com/meesho/python-2.7-Dockerfile new file mode 100644 index 0000000..37f41d0 --- /dev/null +++ b/resources/com/meesho/python-2.7-Dockerfile @@ -0,0 +1,29 @@ +FROM ${buildRegistry}/build/python:2.7-${arch}-0.8.2 as build-system + +COPY ${modules_requirements_file} /app/${modules_requirements_file} + +FROM build-system as intermediate +# add credentials on build +COPY id_github_jenkins /root/.ssh/id_rsa + +RUN ssh-keyscan github.com >> /root/.ssh/known_hosts && \ + chmod -R 600 /root/.ssh/ + +# temp-package python dependencies +RUN pip download -r /app/requirements.txt -d /temp-package/python + +### create the runtime image ### +FROM build-system as runtime + +# install temp-packageed python dependencies +COPY --from=intermediate /temp-package/python /temp-package/python +RUN pip install /temp-package/python/* && \ + rm -rf /temp-package + +WORKDIR /app + +COPY ./ /app/ +<% if (copy_file) { print "COPY copied_files/* $copy_target"} %> +<% add_files.each { print "COPY ${it.path} ${it.target}\n" } %> + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/resources/com/meesho/python-3.10.12-Dockerfile b/resources/com/meesho/python-3.10.12-Dockerfile new file mode 100644 index 0000000..0e62a2c --- /dev/null +++ b/resources/com/meesho/python-3.10.12-Dockerfile @@ -0,0 +1,31 @@ +FROM ${buildRegistry}/build/python:3.10.12-${arch} as build-system + +COPY ${modules_requirements_file} /app/${modules_requirements_file} + +FROM build-system as intermediate +# add credentials on build +COPY id_github_jenkins /root/.ssh/id_rsa + +RUN ssh-keyscan github.com >> /root/.ssh/known_hosts && \ + chmod -R 600 /root/.ssh/ + +# temp-package python dependencies +RUN pip download -r /app/requirements.txt -d /temp-package/python + +### create the runtime image ### +FROM build-system as runtime + +# install temp-packageed python dependencies +COPY --from=intermediate /temp-package/python /temp-package/python +RUN pip install /temp-package/python/* && \ + rm -rf /temp-package && \ + pip install uwsgi && \ + python3 --version + +WORKDIR /app + +COPY ./ /app/ +<% if (copy_file) { print "COPY copied_files/* $copy_target"} %> +<% add_files.each { print "COPY ${it.path} ${it.target}\n" } %> + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/resources/com/meesho/python-3.13-Dockerfile b/resources/com/meesho/python-3.13-Dockerfile new file mode 100644 index 0000000..4f74754 --- /dev/null +++ b/resources/com/meesho/python-3.13-Dockerfile @@ -0,0 +1,31 @@ +FROM ${buildRegistry}/build/python:3.13-${arch} as build-system + +COPY ${modules_requirements_file} /app/${modules_requirements_file} + +FROM build-system as intermediate +# add credentials on build +COPY id_github_jenkins /root/.ssh/id_rsa + +RUN ssh-keyscan github.com >> /root/.ssh/known_hosts && \ + chmod -R 600 /root/.ssh/ + +# temp-package python dependencies +RUN pip download -r /app/requirements.txt -d /temp-package/python + +### create the runtime image ### +FROM build-system as runtime + +# install temp-packageed python dependencies +COPY --from=intermediate /temp-package/python /temp-package/python +RUN pip install /temp-package/python/* && \ + rm -rf /temp-package && \ + pip install uwsgi && \ + python3 --version + +WORKDIR /app + +COPY ./ /app/ +<% if (copy_file) { print "COPY copied_files/* $copy_target"} %> +<% add_files.each { print "COPY ${it.path} ${it.target}\n" } %> + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/resources/com/meesho/python-3.7-Dockerfile b/resources/com/meesho/python-3.7-Dockerfile new file mode 100644 index 0000000..36dd6d8 --- /dev/null +++ b/resources/com/meesho/python-3.7-Dockerfile @@ -0,0 +1,29 @@ +FROM ${buildRegistry}/build/python:3.7-${arch}-0.8.2 as build-system + +COPY ${modules_requirements_file} /app/${modules_requirements_file} + +FROM build-system as intermediate +# add credentials on build +COPY id_github_jenkins /root/.ssh/id_rsa + +RUN ssh-keyscan github.com >> /root/.ssh/known_hosts && \ + chmod -R 600 /root/.ssh/ + +# temp-package python dependencies +RUN pip download -r /app/requirements.txt -d /temp-package/python + +### create the runtime image ### +FROM build-system as runtime + +# install temp-packageed python dependencies +COPY --from=intermediate /temp-package/python /temp-package/python +RUN pip install /temp-package/python/* && \ + rm -rf /temp-package + +WORKDIR /app + +COPY ./ /app/ +<% if (copy_file) { print "COPY copied_files/* $copy_target"} %> +<% add_files.each { print "COPY ${it.path} ${it.target}\n" } %> + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/resources/com/meesho/python-deployment.yaml b/resources/com/meesho/python-deployment.yaml new file mode 100644 index 0000000..3790aa4 --- /dev/null +++ b/resources/com/meesho/python-deployment.yaml @@ -0,0 +1,64 @@ +--- + +app_name: ${app_name} +app_port: ${app_port} +health_check: ${health_check} +module: module_less +bu: ${bu} +team: ${team} +priority: ${priority} +priority_v2: ${priority_v2} +primary_owner: ${primary_owner} +secondary_owner: ${secondary_owner} +initialDelaySeconds: ${initialDelaySeconds} +replica_count: ${replica_count} +environment: + ftr: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + supervisord_config_path: /app/configurations/supervisord/supervisord.conf + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + stg: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + supervisord_config_path: /app/configurations/supervisord/supervisord.conf + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + int: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + supervisord_config_path: /app/configurations/supervisord/supervisord.conf + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + prd: + as_enabled: ${as_enabled} + as_min: ${as_min} + as_max: ${as_max} + supervisord_config_path: /app/configurations/supervisord/supervisord.conf + cpu_limit: ${cpu_limit} + cpu_request: ${cpu_request} + memory_limit: ${memory_limit} + memory_request: ${memory_request} + deploymentStrategy: ${deploymentStrategy} + appMetrics: ${appMetrics} + deployment_args: + \ No newline at end of file diff --git a/resources/com/meesho/python-values.yaml b/resources/com/meesho/python-values.yaml new file mode 100644 index 0000000..205db89 --- /dev/null +++ b/resources/com/meesho/python-values.yaml @@ -0,0 +1,242 @@ +<% if (deploymentStrategy == 'canary'){ print "canary:\n enabled: true\n slackChannel: ${canary.slackChannel}\n enableManualPromotion: ${enableManualPromotion}\n skipAnalysis: ${canary.skipAnalysis}\n service:\n port: 80\n targetPort: ${primary_port}\n progressDeadlineSeconds: ${canary.progressDeadlineSeconds}\n minCanaryReplicas: ${minCanaryReplicas}\n maxCanaryReplicas: ${maxCanaryReplicas}\n analysisInterval: ${canary.analysisInterval}\n analysisThreshold: ${canary.analysisThreshold}\n analysisMaxWeight: ${canary.analysisMaxWeight}\n analysisStepWeight: ${canary.analysisStepWeight}\n analysisMetrics:\n thresholdRangeMin: ${canary.analysisMetrics.thresholdRangeMin}\n interval: ${canary.analysisMetrics.interval}\n" } else { print "canary:\n enabled: false\n" } %>repoName: ${repo_name} +cron: + enabled: false + serviceAccount: + enabled: false +applicationName: ${app_name} +appType: ${dockerBuildVersion} +autoscaling: + enabled: ${as_enabled} + maxReplicas: ${as_max} + minReplicas: ${as_min} + pollingInterval: ${as_poll} + scaledown: + policies: + - periodseconds: ${as_down_period} + type: Pods + value: ${as_down_pod_count} + selectpolicy: Min + stabilizationWindowSeconds: ${as_down_stable_window} + scaleup: + policies: + - periodseconds: ${as_up_period} + type: Pods + value: ${as_up_pod_count} + - periodseconds: ${as_up_period} + type: Percent + value: ${as_up_pod_percentage} + selectpolicy: Max + stabilizationWindowSeconds: ${as_up_stable_window} +<% if(!triggers){print " triggers:\n - metadata:\n value: \"${as_trigger_value}\"\n metricType: ${as_trigger_type}\n type: ${as_trigger_metric}"} else {println ' triggers:'; for(val in triggers){if(val instanceof Map){ val.each{k,v -> if (v instanceof Map) { println ' - '+k+':'; v.each{a,b -> if(b.isNumber()){println " ${a}: \"${b}\"" } else {println " ${a}: ${b}" }}} else {println" ${k}: ${v}"}}}}} %> +deployment: + enabled: <% if (kind == "deployment" || kind == "Deployment") { print "true" } else { print "false" } %> + updateStrategy: + strategy: + rollingUpdate: + maxUnavailable: 0% + maxSurge: <% print "${maxSurge}%" %> + type: RollingUpdate + affinity: {} + podDistributionSkew: ${podDistributionSkew} + env: + - name: SUPERVISORD_CONFIG_FILE + value: ${supervisord_config_path} + envFrom: + secretRef: ${app_name} + image: + pullPolicy: IfNotPresent + pullSecret: "" + repository: ${registry}/${environment}/${build_team}/<% print module=='module_less'?repo_name.toLowerCase():repo_name.toLowerCase()+'/'+module %> + tag: ${tag} +<% if(lifecycle) { println ' lifecycle:\n preStop:\n exec:\n command:'; for(val in lifecycle.preStop.exec.command){ println " - $val" }} else {print " lifecycle:\n preStop:\n exec:\n command:\n - /bin/bash\n - -c\n - kill -SIGQUIT 1 ; /bin/sleep 60"} %> + minReadySeconds: 10 + podAnnotations: + <% if (appMetrics) { print 'prometheus.io/path: /actuator/prometheus' %> + <% print 'prometheus.io/port: "'+app_port+'"' %> + <% print 'prometheus.io/scrape: "true"'} %> +<% if (pod_annotations) {pod_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + ports: + - containerPort: ${app_port} + name: http + protocol: TCP + - containerPort: 9901 + name: metric + protocol: TCP + probes: + liveness: + failureThreshold: ${liveness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${liveness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${liveness_success_threshold} + timeoutSeconds: ${liveness_timeout_seconds} + readiness: + failureThreshold: ${readiness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${readiness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${readiness_success_threshold} + timeoutSeconds: ${readiness_timeout_seconds} + replicaCount: ${replica_count} + resources: + limits: + cpu: ${cpu_limit} + memory: ${memory_limit}i + requests: + cpu: ${cpu_request} + memory: ${memory_request}i + revisionHistoryLimit: 6 +<% if(serviceAccount){println ' serviceAccount:\n annotations:';serviceAccount.annotations.each{k,v -> println " ${k}: ${v}"};println " enabled: ${serviceAccount.enabled}"} else {print ' serviceAccount:\n annotations: null\n enabled: false'} %> + nodeSelector: + ${nodeSelector}: ${nodeSelectorValue} +<% if (hostAliases){println ' hostAliases:';for(arr in hostAliases){println ' - ip: '+arr.get("ip"); println ' hostnames:'; for (ele in arr.get("hostnames")) { println ' - '+ele } }} %> + tolerations: + - effect: NoSchedule + key: ${nodeSelector} + operator: Equal + value: ${nodeSelectorValue} +externalSecret: + annotations: +<% if (external_secrets_annotations) {external_secrets_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true + path: meesho/${vault_env}/${bu}/${team}/${app_name} + version: ${tag} + name: ${env_ns}-${app_name} + target: ${app_name} +fullnameOverride: "" +replicaCount: ${replica_count} +ingress: + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/ssl-redirect: "false" + enabled: true +<% if(grpc_host){println " grpc_hosts:\n - host: ${grpc_host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else if (grpc_hosts) {println " grpc_hosts:";for(host_arr in grpc_hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> +<% if(host){println " hosts:\n - host: ${host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else {println " hosts:";for(host_arr in hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> ingressClassName: ${ingress_class} + servicePort: http + enableWebsocket: ${enableWebsocket} + slowStart: + enabled: <% if (slowStartWindow) { print "true" } else { print "false" } %> + window: <% if (slowStartWindow) { print "${slowStartWindow}" } else { print "120s" } %> + aggression: <% if (slowStartAggression) { print "${slowStartAggression}" } else { print "1.0" } %> + minPercent: <% if (slowStartMinPercent) { print "${slowStartMinPercent}" } else { print "10" } %> +jmxconfig: + enabled: true +labels: + priority: <% print priority?:'p1' %> + priority_v2: <% print priority_v2?:'cp3' %> + primary_owner: ${primary_owner} + secondary_owner: ${secondary_owner} + env: ${environment_norm} + team: ${team_norm} + bu: ${bu_norm} + <% if (service_type_norm) { println "service_type: ${service_type_norm}" } %> + commit_id: ${commit_id} +nameOverride: "" +namespace: ${env_ns}-${app_name} +podDisruptionBudget: + enabled: false + maxUnavailable: 100% + minAvailable: ${pdbMinAvailable} +podSecurityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsUser: 65534 +service: +<% if (service_annotations) { println " annotations:"; service_annotations.each{k,v -> println " ${k}: \"${v}\""};} else { println " annotations: null" } %> + enabled: true + ports: + - name: http + port: 80 + protocol: TCP + targetPort: ${app_port} + type: ClusterIP +statefulset: + enabled: <% if (kind == "statefulset" || kind == "StatefulSet" || kind == "statefulSet") { print "true" } else { print "false" } %> + updateStrategy: <% if (statefulset.updateStrategy == "RollingUpdate" || statefulset.updateStrategy == "rollingUpdate" || statefulset.updateStrategy == "rollingupdate") { print "RollingUpdate" } else { print "null" } %> + volumeType: ${statefulset.volumeType} +<% if(statefulset.volumeType == "static" || statefulset.volumeType == "Static") { println " staticVolume:"; statefulset.staticVolume.each{k,v -> println " ${k}: ${v}"};} else {print " staticVolume:\n accessMode: ReadWriteMany\n mountPath: /opt/data\n size: 5Gi\n storageClass: meesho-gp2\n volumeHandle: fs-0e2a97a38b01857d1::fsap-07b146e3bc7aef280\n csiDriver: efs.csi.aws.com"} %> +<% if(statefulset.volumeType == "dynamic" || statefulset.volumeType == "Dynamic") { println " dynamicVolume:"; statefulset.dynamicVolume.each{k,v -> println " ${k}: ${v}"};} else {print " dynamicVolume:\n accessMode: ReadWriteMany\n mountPath: /opt/data\n size: 5Gi\n storageClass: meesho-gp2"} %> +createContourGateway: <% if (createContourGateway) { print "${createContourGateway}" } else { print "false" } %> +contourResponseTimeout: ${contourResponseTimeout} +podtemplate: + affinity: {} + env: + - name: SUPERVISORD_CONFIG_FILE + value: ${supervisord_config_path} + envFrom: + secretRef: ${app_name} + image: + pullPolicy: IfNotPresent + pullSecret: "" + repository: ${registry}/${environment}/${build_team}/<% print module=='module_less'?repo_name.toLowerCase():repo_name.toLowerCase()+'/'+module %> + tag: ${tag} +<% if(lifecycle) { println ' lifecycle:\n preStop:\n exec:\n command:'; for(val in lifecycle.preStop.exec.command){ println " - $val" }} else {print " lifecycle:\n preStop:\n exec:\n command:\n - /bin/bash\n - -c\n - kill -SIGQUIT 1 ; /bin/sleep 60"} %> + minReadySeconds: 10 + podAnnotations: + <% if (appMetrics) { print 'prometheus.io/path: /actuator/prometheus' %> + <% print 'prometheus.io/port: "'+app_port+'"' %> + <% print 'prometheus.io/scrape: "true"'} %> +<% if (pod_annotations) {pod_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + ports: + - containerPort: ${app_port} + name: http + protocol: TCP + - containerPort: 9901 + name: metric + protocol: TCP + probes: + liveness: + failureThreshold: ${liveness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${liveness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${liveness_success_threshold} + timeoutSeconds: ${liveness_timeout_seconds} + readiness: + failureThreshold: ${readiness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${readiness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${readiness_success_threshold} + timeoutSeconds: ${readiness_timeout_seconds} + replicaCount: ${replica_count} + resources: + limits: + cpu: ${cpu_limit} + memory: ${memory_limit}i + requests: + cpu: ${cpu_request} + memory: ${memory_request}i + revisionHistoryLimit: 6 + serviceAccount: + annotations: null + enabled: false + nodeSelector: + ${nodeSelector}: ${nodeSelectorValue} +<% if (hostAliases){println ' hostAliases:';for(arr in hostAliases){println ' - ip: '+arr.get("ip"); println ' hostnames:'; for (ele in arr.get("hostnames")) { println ' - '+ele } }} %> + tolerations: + - effect: NoSchedule + key: ${nodeSelector} + operator: Equal + value: ${nodeSelectorValue} +appConfig: + enabled: <% if(appConfigEnabled) { print "true" } else { print "false"} %> + env: ${environment} +<% if(appConfigEnabled) {%> + staticAppConfig: + data: | +${staticAppConfigData.trim().replaceAll("(?m)^", " ")} + dynamicAppConfig: + data: | +${dynamicAppConfigData.trim().replaceAll("(?m)^", " ")} +<% } +%> diff --git a/resources/com/meesho/rust-Dockerfile b/resources/com/meesho/rust-Dockerfile new file mode 100644 index 0000000..78fbff7 --- /dev/null +++ b/resources/com/meesho/rust-Dockerfile @@ -0,0 +1,58 @@ +FROM ${buildRegistry}/build/rust:${version} AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + protobuf-compiler \ + cmake \ + libprotobuf-dev \ + libssl-dev \ + pkg-config \ + openssh-client \ + ca-certificates \ + libsasl2-2 \ + libsasl2-dev \ + clang \ + libclang-dev \ +<% build_packages.each { print " ${it} \\\n" } %> + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/src/app + + +COPY id_github_jenkins /root/.ssh/id_rsa + +RUN ssh-keyscan github.com >> /root/.ssh/known_hosts && \ + chmod -R 600 /root/.ssh/ && \ + git config --global url."git@github.com:".insteadOf "https://github.com/" + +# Copy source code +COPY . . + +# Build release binary for the native architecture (amd64) +RUN cargo build --workspace --release + +# Clean up SSH key from builder stage for security +RUN rm -rf /root/.ssh/ + +FROM asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/build/debian:trixie-slim + +# Install runtime dependencies if your app needs them (e.g., SSL) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libssl3t64 \ + ca-certificates \ + libsasl2-2 \ +<% runtime_packages.each { print " ${it} \\\n" } %> + && rm -rf /var/lib/apt/lists/* + +# Set workdir for the binary +WORKDIR /app + +# Copy the compiled amd64 binary from the builder +COPY --from=builder /usr/src/app/target/release/${binary_name} ./server +<% add_files.each { print "COPY ${it.path} ${it.target}\n" } %> + +# Expose port if your app listens on it +EXPOSE 8080 + +# Start the application +CMD ["./server"] diff --git a/resources/com/meesho/rust-values.yaml b/resources/com/meesho/rust-values.yaml new file mode 100644 index 0000000..4b26f08 --- /dev/null +++ b/resources/com/meesho/rust-values.yaml @@ -0,0 +1,206 @@ +<% if (deploymentStrategy == 'canary'){ print "canary:\n enabled: true\n slackChannel: ${canary.slackChannel}\n enableManualPromotion: ${enableManualPromotion}\n skipAnalysis: ${canary.skipAnalysis}\n service:\n port: 80\n targetPort: ${(grpc_host || grpc_hosts) ? app_port : primary_port}\n progressDeadlineSeconds: ${canary.progressDeadlineSeconds}\n minCanaryReplicas: ${minCanaryReplicas}\n maxCanaryReplicas: ${maxCanaryReplicas}\n analysisInterval: ${canary.analysisInterval}\n analysisThreshold: ${canary.analysisThreshold}\n analysisMaxWeight: ${canary.analysisMaxWeight}\n analysisStepWeight: ${canary.analysisStepWeight}\n analysisMetrics:\n thresholdRangeMin: ${canary.analysisMetrics.thresholdRangeMin}\n interval: ${canary.analysisMetrics.interval}\n" } else { print "canary:\n enabled: false\n" } %> +repoName: ${repo_name} +cron: + enabled: false + serviceAccount: + enabled: false +applicationName: ${app_name} +autoscaling: + enabled: ${as_enabled} + maxReplicas: ${as_max} + minReplicas: ${as_min} + pollingInterval: ${as_poll} + scaledown: + policies: + - periodseconds: ${as_down_period} + type: Pods + value: ${as_down_pod_count} + selectpolicy: Min + stabilizationWindowSeconds: ${as_down_stable_window} + scaleup: + policies: + - periodseconds: ${as_up_period} + type: Pods + value: ${as_up_pod_count} + - periodseconds: ${as_up_period} + type: Percent + value: ${as_up_pod_percentage} + selectpolicy: Max + stabilizationWindowSeconds: ${as_up_stable_window} +<% if(!triggers){print " triggers:\n - metadata:\n value: \"${as_trigger_value}\"\n metricType: ${as_trigger_type}\n type: ${as_trigger_metric}"} else {println ' triggers:'; for(val in triggers){if(val instanceof Map){ val.each{k,v -> if (v instanceof Map) { println ' - '+k+':'; v.each{a,b -> if(b.isNumber()){println " ${a}: \"${b}\"" } else {println " ${a}: ${b}" }}} else {println" ${k}: ${v}"}}}}} %> +replicaCount: ${replica_count} +deployment: + affinity: {} + podDistributionSkew: ${podDistributionSkew} +<% if (deployment_args){println ' args:'; for (arg in deployment_args){ println ' - '+arg}} %> + command: + - ${command} + enabled: true + env: + - name: GOMAXPROCS + value: ${activeProcessorCount} + envFrom: + secretRef: ${app_name} + image: + pullPolicy: IfNotPresent + pullSecret: "" + repository: ${registry}/${environment}/${build_team}/<% print module=='module_less'?repo_name.toLowerCase():repo_name.toLowerCase()+'/'+module %> + tag: ${tag} +<% if(lifecycle) { println ' lifecycle:\n preStop:\n exec:\n command:'; for(val in lifecycle.preStop.exec.command){ println " - $val" }} else {print " lifecycle:\n preStop:\n exec:\n command:\n - /bin/bash\n - -c\n - kill -SIGQUIT 1 ; /bin/sleep 120\n "} %> + minReadySeconds: 10 + podAnnotations: + <% if (appMetrics) { print 'prometheus.io/path: /actuator/prometheus' %> + <% print 'prometheus.io/port: "'+app_port+'"' %> + <% print 'prometheus.io/scrape: "true"'} %> + <% if (nodeSelector.contains("arm64") && (environment=="int" || environment=="prd")) { print 'telegraf.influxdata.com/image: 847438129436.dkr.ecr.ap-southeast-1.amazonaws.com/telegraf:1.24.4-arm64' } %> +<% if (pod_annotations) {pod_annotations.each{k,v -> println " ${k}: '${v}'"};} else {print ''} %> + ports: + - containerPort: ${app_port} + name: http + protocol: TCP + - containerPort: 8880 + name: metric + protocol: TCP + probes: + liveness: + failureThreshold: ${liveness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${liveness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${liveness_success_threshold} + timeoutSeconds: ${liveness_timeout_seconds} + readiness: + failureThreshold: ${readiness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${readiness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${readiness_success_threshold} + timeoutSeconds: ${readiness_timeout_seconds} + resources: + limits: + cpu: ${cpu_limit} + memory: ${memory_limit}i + requests: + cpu: ${cpu_request} + memory: ${memory_request}i + revisionHistoryLimit: 6 +<% if(serviceAccount){println ' serviceAccount:\n annotations:';serviceAccount.annotations.each{k,v -> println " ${k}: ${v}"};println " enabled: ${serviceAccount.enabled}"} else {print ' serviceAccount:\n annotations: null\n enabled: false'} %> + nodeSelector: + ${nodeSelector}: ${nodeSelectorValue} +<% if (hostAliases){println ' hostAliases:';for(arr in hostAliases){println ' - ip: '+arr.get("ip"); println ' hostnames:'; for (ele in arr.get("hostnames")) { println ' - '+ele } }} %> + tolerations: + - effect: NoSchedule + key: ${nodeSelector} + operator: Equal + value: ${nodeSelectorValue} + # - effect: NoSchedule + # key: kubernetes.io/arch + # operator: Equal + # value: arm64 + updateStrategy: + strategy: +<% if (deploymentStrategy == 'recreate') { print " type: Recreate"} else { print " type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 0%\n maxSurge: ${maxSurge}%" } %> +externalSecret: + annotations: +<% if (external_secrets_annotations) {external_secrets_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true + path: meesho/${vault_env}/${bu}/${team}/${app_name} + version: ${tag} + name: ${env_ns}-${app_name} + target: ${app_name} +fullnameOverride: "" +ingress: + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/ssl-redirect: "false" + nginx.ingress.kubernetes.io/server-snippet: | + location ~* "^/api/1.0/search/recent" { + default_type application/json; + return 200 '{"recent_searches": [],"recent_suggestions": [],"limit": 5,"autosuggest_use_recent": true,"show_recent_header": false}'; + } + location ~* "^/api/1.0/anonymous/search/recent" { + default_type application/json; + return 200 '{"recent_searches": [],"recent_suggestions": [],"limit": 5,"autosuggest_use_recent": true,"show_recent_header": false}'; + } + location ~* "^/search-queries/recent" { + default_type application/json; + return 200 '{"recent_searches": [],"recent_suggestions": [],"limit": 5,"autosuggest_use_recent": true,"show_recent_header": false}'; + } +<% if (ingress_annotations) {ingress_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true +<% if(grpc_host){println " grpc_hosts:\n - host: ${grpc_host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else if (grpc_hosts) {println " grpc_hosts:";for(host_arr in grpc_hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> +<% if(host){println " hosts:\n - host: ${host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else {println " hosts:";for(host_arr in hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> ingressClassName: ${ingress_class} + servicePort: http + enableWebsocket: ${enableWebsocket} + slowStart: + enabled: <% if (slowStartWindow) { print "true" } else { print "false" } %> + window: <% if (slowStartWindow) { print "${slowStartWindow}" } else { print "120s" } %> + aggression: <% if (slowStartAggression) { print "${slowStartAggression}" } else { print "1.0" } %> + minPercent: <% if (slowStartMinPercent) { print "${slowStartMinPercent}" } else { print "10" } %> +jmxconfig: + enabled: false +labels: + priority: <% print priority?:'p1' %> + priority_v2: <% print priority_v2?:'cp3' %> + primary_owner: ${primary_owner} + secondary_owner: ${secondary_owner} + env: ${environment_norm} + team: ${team_norm} + bu: ${bu_norm} + <% if (service_type_norm) { println "service_type: ${service_type_norm}" } %> + commit_id: ${commit_id} +nameOverride: "" +namespace: ${env_ns}-${app_name} +podDisruptionBudget: + enabled: <% if (pdbMaxUnavailable) { print "true" } else { print "false" } %> + maxUnavailable: ${pdbMaxUnavailable} + minAvailable: ${pdbMinAvailable} +podSecurityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsUser: 65534 +service: +<% if (service_annotations) { println " annotations:"; service_annotations.each{k,v -> println " ${k}: \"${v}\""};} else { println " annotations: null" } %> + enabled: true +<% if (addon_ports) { for (p in addon_ports) { println " addons:"; println " - name: ${p.name}"; println " targetPort: ${p.targetPort}"; println " type: ${p.type}"; }} else { println " addon_ports: []" } %> +<% if (grpc_port && app_port && (grpc_host || grpc_hosts)) { + println " ports:" + println " - name: http" + println " port: 80" + println " protocol: TCP" + println " targetPort: ${app_port}" + println " grpc_ports:" + println " - name: grpc" + println " port: 80" + println " protocol: TCP" + println " targetPort: ${grpc_port}" +} else { + println " ports:" + println " - name: http" + println " port: 80" + println " protocol: TCP" + println " targetPort: ${primary_port}" +} +%> + type: ClusterIP + +createContourGateway: <% if (createContourGateway) { print "${createContourGateway}" } else { print "false" } %> +contourResponseTimeout: ${contourResponseTimeout} + +appConfig: + enabled: <% if(appConfigEnabled) { print "true" } else { print "false"} %> + env: ${environment} +<% if(appConfigEnabled) {%> + staticAppConfig: + data: | +${staticAppConfigData.trim().replaceAll("(?m)^", " ")} + dynamicAppConfig: + data: | +${dynamicAppConfigData.trim().replaceAll("(?m)^", " ")} +<% } +%> diff --git a/resources/com/meesho/validate_configs.py b/resources/com/meesho/validate_configs.py new file mode 100644 index 0000000..6b79d32 --- /dev/null +++ b/resources/com/meesho/validate_configs.py @@ -0,0 +1,1233 @@ +import re +import yamale, validators +from collections.abc import MutableMapping +from deepmerge import merger +from yamale.validators import DefaultValidators, Validator +import yaml +import copy +import os +from collections import Counter +import tempfile +import json +from urllib.parse import urlparse + +from detect_secrets import SecretsCollection +from detect_secrets.settings import transient_settings + +from deepmerge import Merger + +merger = Merger( + [(dict, ["merge"])], + ["override"], + ["override"] +) + +deployable_section_format = "" + +invalid_patterns = { + 'stg': ['int.meesho.int', 'prd.meesho.int'], + 'int': ['stg.meesho.int'], + 'prd': ['int.meesho.int', 'stg.meesho.int'] +} +valid_zk_patterns = { +'dev': ['dev.meesho.int:2181'], + 'stg': ['dev.meesho.int:2181'], + 'int': ['int.meesho.int:2181'], + 'prd': ['config.prd.meesho.int:2181','mlp.prd.meesho.int:2181','comms.prd.meesho.int:2181'] +} + +whitelists_for_preprod = ['consumer', 'scheduler', 'cron'] + +db_patterns = [ + # PostgreSQL + r'^postgresql://(?:\S+):(?:\S+)@(?:\S+):\d+/\S+$', # PostgreSQL (standard) + r'^jdbc:postgresql://(?:\S+):(?:\S+)@(?:\S+):\d+/\S+$', # PostgreSQL (JDBC) + + # MySQL + r'^mysql://(?:\S+):(?:\S+)@(?:\S+):\d+/\S+$', # MySQL (standard) + r'^jdbc:mysql://*', # MySQL (JDBC) with optional query parameters + + # MongoDB + r'^mongodb://(?:\S+):(?:\S+)@(?:\S+):\d+/\S+$', # MongoDB (standard) + r'^mongodb\+srv://(?:\S+):(?:\S+)@(?:\S+)(?:/\S+)?$', # MongoDB (SRV) + + # SQLite + r'^sqlite://(?:\S+)$', # SQLite + + # Redis + r'^redis://(?:\S+):\d+$' # Redis (host:port) +] + +config_files_props = [ + { + 'name': 'application-prd.yml', + 'type': 'static', + 'schema-file-name': 'application-schema.yml', + 'env': 'prd' + }, + { + 'name': 'application-int.yml', + 'type': 'static', + 'schema-file-name': 'application-schema.yml', + 'env': 'int' + }, + { + 'name': 'application-stg.yml', + 'type': 'static', + 'schema-file-name': 'application-schema.yml', + 'env': 'stg' + }, + { + 'name': 'application-dev.yml', + 'type': 'static', + 'schema-file-name': 'application-schema.yml', + 'env': 'dev' + }, + { + 'name': 'application-dyn-prd.yml', + 'type': 'dynamic', + 'schema-file-name': 'application-schema.yml', + 'env': 'prd' + }, + { + 'name': 'application-dyn-int.yml', + 'type': 'dynamic', + 'schema-file-name': 'application-schema.yml', + 'env': 'int' + }, + { + 'name': 'application-dyn-stg.yml', + 'type': 'dynamic', + 'schema-file-name': 'application-schema.yml', + 'env': 'stg' + }, + { + 'name': 'application-dyn-dev.yml', + 'type': 'dynamic', + 'schema-file-name': 'application-schema.yml', + 'env': 'dev' + } +] + + +class NonRequiredValidator(Validator): + """ Wrapper around existing validators to make fields optional. """ + + def __init__(self, base_validator, *args, **kwargs): + super().__init__(*args, **kwargs) + self.base_validator = base_validator + + def is_valid(self, value): + """ Allow missing values by returning True when value is None """ + if value is None: + return True + return self.base_validator.is_valid(value) + + def to_python(self, value): + return self.base_validator.to_python(value) + +class NumOrStringNum(Validator): + tag = 'num' + + def _is_valid(self, value): + if isinstance(value, (int, float)): + return True + if isinstance(value, str): + try: + float(value) # check if string can be parsed as number + return True + except ValueError: + return False + return False + + def _format(self, value): + """Converts a valid string representation of a number into int or float.""" + if isinstance(value, str): + try: + # convert to int if it's whole number, else float + f = float(value) + return int(f) if f.is_integer() else f + except ValueError: + return value + return value + +class JsonStr(Validator): + tag = 'json_str' # Custom tag to use in the schema + + def _is_valid(self, value): + # Check if value is already a dict or list (already parsed JSON) + if isinstance(value, (dict, list)): + return True + # Check if value is a string that can be parsed as JSON + if isinstance(value, str): + try: + json.loads(value) + return True + except (json.JSONDecodeError, TypeError): + return False + return False + + def _format(self, value): + """Converts a valid JSON string into a parsed JSON object.""" + if isinstance(value, str): + try: + return json.loads(value) + except (json.JSONDecodeError, TypeError): + pass + return value + +class IntOrStringInt(Validator): + tag = 'int' # Custom tag to use in the schema + + def _is_valid(self, value): + # Check if value is an integer + if isinstance(value, int): + return True + # Check if value is a string that can be converted to an integer + if isinstance(value, str): + try: + int(value) + return True + except ValueError: + return False + return False + + def _format(self, value): + """Converts a valid string representation of an integer into an integer.""" + if isinstance(value, str): + try: + return int(value) + except ValueError: + pass + return value + + +class BoolOrStringBool(Validator): + tag = 'bool' + + def _is_valid(self, value): + if isinstance(value, bool): + return True + if isinstance(value, str): + value = value.lower() + if value in ['true', 'false']: + return True + return False + + def _format(self, value): + """Converts a valid string representation of a boolean into a boolean.""" + if isinstance(value, str): + value = value.lower() + if value == 'true': + return True + elif value == 'false': + return False + return value + + +class StringCustom(Validator): + tag = 'str' + + def __init__(self, max=None, required=False): + super().__init__(required=required) + self.max_length = max + self.required = required + + def _is_valid(self, value): + errors = [] + + if self.required and value is None: + errors.append(self.fail("Value is required but missing.")) + + if not isinstance(value, (str, int, bool)): + errors.append(self.fail(f"Invalid type '{type(value).__name__}'. Expected str, int, or bool.")) + + if isinstance(value, str) and self.max_length is not None and len(value) > self.max_length: + errors.append(self.fail(f"Value exceeds maximum length of {self.max_length} characters.")) + + return errors # Return list of errors instead of True/False + + def fail(self, message): + """Override to define a custom fail message""" + return f"Validation Error: {message}" + + def validate(self, value): + """ + Override validate to return a list of validation errors. + """ + errors = self._is_valid(value) # Get errors from _is_valid + + # Validate constraints (if any exist) + for constraint in self._constraints_inst: + error = constraint.is_valid(value) + if error: + if isinstance(error, list): + errors.extend(error) + else: + errors.append(error) + + return errors # Return full list of validation errors + + def is_valid(self, value): + return not self.validate(value) # Returns True if no errors + + def _format(self, value): + if isinstance(value, int): + value = str(value) + if isinstance(value, str) and self.max_length is not None and len(value) > self.max_length: + return value[:self.max_length] + return value + + + +def is_db_url(key, value): + """ + Check if the given key-value pair is a valid database URL or name. + + Parameters: + key_value_pair (tuple): A tuple containing the key and value to check. + + Returns: + bool: True if the value is a valid database URL or name, False otherwise. + """ + + # Regex patterns for different database URL formats + db_patterns = [ + r'^(postgresql://|mysql://|sqlite://|mongodb://|oracle://|mssql://|cockroachdb://|redis://|cassandra://|neo4j://|jdbc:mysql://).+', + r'^[a-zA-Z0-9_]+$', # Just a database name (alphanumeric + underscores) + r'^[a-zA-Z0-9_]+\.([a-zA-Z0-9_]+)$' # Database name with an optional prefix (e.g., schema) + ] + + # Check if the value matches any of the patterns + for pattern in db_patterns: + if re.match(pattern, value): + return True + + return False + + +class Reader: + """ + This class provides methods to read the yml configuration file which has sections. + File structure should be as follows: + + default_Data: + + --- + spring: + profiles: sections_1 + sections_1_data + + --- + spring: + profiles: sections_2 + sections_2_data + + --- + spring: + profiles: sections_3 + sections_3_data + """ + + def __init__(self, config_file_path: str): + self.config_file_path = config_file_path + + def read_config_file(self): + """ + Read the config file and return the data in the form of dictionary. + """ + data = yamale.make_data(self.config_file_path) + return data + + # Function to process each YAML file + def read_without_sections(self): + with open(self.config_file_path, 'r') as file: + try: + data = yaml.safe_load(file) + return data + except yaml.YAMLError as exc: + print(f"Error reading {self.config_file_path}: {exc}", flush=True) + return None + + +class DictUtils: + @staticmethod + def sub_dict(keys: list[str], data: dict) -> dict: + """ + Function to extract the dictionary from the keys + :param keys: keys to extract + :param data: dictionary of data + :return: dictionary with the keys + """ + return {key: data[key] for key in keys} + + @staticmethod + def clean_up(data, keys): + """ + Delete the specified nested key and clean up empty parent keys. + + :param data: The dictionary to clean up. + :param keys: A list of keys specifying the path to the nested key. + """ + if not keys: + return + + current_key = keys[0] + if len(keys) == 1: + data.pop(current_key, None) + else: + next_level = data.get(current_key, None) + if next_level is not None and isinstance(next_level, dict): + DictUtils.clean_up(next_level, keys[1:]) + if not next_level: + data.pop(current_key, None) + + @staticmethod + def remove_common_keys(dict_of_dicts, common_data): + for key in dict_of_dicts: + for common_key in common_data.keys(): + if common_key in dict_of_dicts[key]: + del dict_of_dicts[key][common_key] + return dict_of_dicts + + @staticmethod + def find_common_values(dict_of_dicts): + dict_list = list(dict_of_dicts.values()) + common_values = dict_list[0].copy() + for d in dict_list[1:]: + common_values = {k: v for k, v in common_values.items() if k in d and d[k] == v} + return common_values + + @staticmethod + def remove_keys(dictionary: dict, keys: list): + for key in keys: + if key in dictionary: + del dictionary[key] + + @staticmethod + def extract_dict_with_chosen_keys(dictionary: dict, keys: list): + new_dict = dict() + for key in keys: + if key in dictionary: + new_dict[key] = dictionary[key] + return new_dict + + @staticmethod + def extract_common_data_outside(nested_dict: dict): + common_values = DictUtils.find_common_values(nested_dict) + dict_list = DictUtils.remove_common_keys(nested_dict, common_values) + # Assuming common_values and dict_list are defined + merged_dict = common_values.copy() # Start with keys and values from common_values + merged_dict.update(dict_list) # Adds keys and values from each dictionary in dict_list to merged_dict + return merged_dict + + @staticmethod + def extract_common_majority(dict_of_dicts): + dict_keys = list(dict_of_dicts.keys()) + + if not dict_keys: + return dict_of_dicts + + # Step 1: Identify common keys across all inner dictionaries + common_keys = set(dict_of_dicts[dict_keys[0]].keys()) + for key in dict_keys[1:]: + common_keys.intersection_update(dict_of_dicts[key].keys()) + + common_dict = {} + + for key in common_keys: + if key in ["spring~application~name", "spring~profiles", "spring~config~activate~on-profile","deployable-name"]: + continue + serialized_values = [json.dumps(dict_of_dicts[d][key], sort_keys=True) for d in dict_keys] + most_common_serialized_value, count = Counter(serialized_values).most_common(1)[0] + most_common_value = json.loads(most_common_serialized_value) + common_dict[key] = most_common_value + for d in set(dict_keys): + if json.dumps(dict_of_dicts[d][key], sort_keys=True) == most_common_serialized_value: + del dict_of_dicts[d][key] + dict_of_dicts["common"] = common_dict + return dict_of_dicts + + @staticmethod + def extract_common(dict_of_dicts, common_data: dict): + """ + Extract common data from the dictionary of dictionaries and if some keys are there which are common in + dict_of_dicts but not present in common_data then for them we will use the majority value + :param dict_of_dicts: dict: deployable -> config_key -> config_value + :param common_data: dict: config_key -> config_value + :return: + """ + for key, value in common_data.items(): + for deployable in dict_of_dicts.keys(): + if key in dict_of_dicts[deployable] and dict_of_dicts[deployable][key] == value: + dict_of_dicts[deployable].pop(key) + dict_of_dicts = DictUtils.extract_common_majority(dict_of_dicts) + dict_of_dicts["common"] = merger.merge(common_data, dict_of_dicts.get("common", {})) + return dict_of_dicts + + +class JsonFileUtils: + + @staticmethod + def get_keys_by_line_numbers(file_path: str, line_numbers: list[int]) -> list[str]: + keys = [] + with open(file_path, 'r') as file: + for i, line in enumerate(file, start=1): + if i in line_numbers: + # Extract the key from the line + key = line.strip().split(':')[0].replace('"', '').strip() + keys.append(key) + return keys + + @staticmethod + def write_dict_to_json_file(data: dict, file_path: str): + with open(file_path, "w") as json_file: + json.dump(data, json_file, indent=4) + + +class SecretFinder: + + def __init__(self, file_path: str, is_exclude_keyword_detector: bool = False): + self.file_path = file_path + self.secrets = SecretsCollection() + self.settings = { + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + # { + # "name": "Base64HighEntropyString", + # "limit": 4.5 + # }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, + # { + # "name": "HexHighEntropyString", + # "limit": 3.0 + # }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "OpenAIDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TelegramBotTokenDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ] + } + if not is_exclude_keyword_detector: + self.settings["plugins_used"].append({ + "name": "KeywordDetector", + "keyword_exclude": "" + } + ) + + def fetch_secrets(self) -> dict: + """ + Function to check for secretes in the filex + :return: line numbers of the secretes in the file + """ + with transient_settings(self.settings): + self.secrets.scan_file(self.file_path) + + return self.secrets.json().get(self.file_path, {}) + + +class SecretHelper: + def __init__(self): + pass + + @staticmethod + def extract_secrets_positions(secrets: dict) -> list[int]: + """ + Function to extract the line numbers of the secrets + :param secrets: secret find result from the secret finder + :return: line numbers of the secretes in the file + """ + line_numbers = set() + if len(secrets) > 0: + for secret in secrets: + line_numbers.add(secret['line_number']) + return sorted(list(line_numbers)) + + @staticmethod + def is_secret_present(file_path: str, is_exclude_keyword_detector: bool = False) -> bool: + """ + Function to check for secretes in the file + :param is_exclude_keyword_detector: if we want to exclude keyword detector + :param file_path: config file_path + :return: error_message and status of the check + """ + secrets = SecretFinder(file_path, is_exclude_keyword_detector).fetch_secrets() + if len(secrets) != 0: + print(json.dumps(secrets, indent=2), flush=True) + secrets_line_numbers = SecretHelper.extract_secrets_positions(secrets) + return len(secrets_line_numbers) > 0 + + @staticmethod + def fetch_secrets_keys_from_json(file_path: str) -> list[str]: + """ + Function to fetch the secret keys from the json file + :return: list of secret keys + """ + secrets = SecretFinder(file_path).fetch_secrets() + secrets_line_numbers = SecretHelper.extract_secrets_positions(secrets) + secret_keys = JsonFileUtils.get_keys_by_line_numbers(file_path, secrets_line_numbers) + return secret_keys + + @staticmethod + def filter_secrets_from_dict(data: dict, secret_keys: list[str]) -> dict: + """ + Function to filter the secrets from the dictionary + :param secret_keys: secret keys + :param data: dictionary of data + :return: dictionary without the secrets + """ + return DictUtils.sub_dict(secret_keys, data) + + @staticmethod + def find_secrets_in_json_files(directory_path: str) -> dict: + + file_to_secrets_dict = dict() + + for filename in os.listdir(directory_path): + if filename.endswith(".json"): + file_path = os.path.join(directory_path, filename) + secrets = SecretHelper.fetch_secrets_keys_from_json(file_path) + file_to_secrets_dict[filename] = secrets + + return file_to_secrets_dict + + +class ParsingHelper: + + @staticmethod + def get_default_section(reader: Reader): + return reader.read_config_file()[0][0] + + @staticmethod + def get_section_data(reader: Reader, section_name: str) -> dict: + data = reader.read_config_file() + default_section = data[0][0] + is_first = True + for section in data: + if is_first: + is_first = False + continue + if (section[0].get('spring', {}).get('profiles', None) == section_name or + section[0].get('spring', {}).get('config', {}).get("activate", {}).get("on-profile", + None) == section_name): + requested_section = default_section + requested_section['spring']['profiles'] = section[0].get('spring', {}).get('profiles', None) + requested_section['spring']['config']['activate']['on-profile'] = section[0].get('spring', {}).get( + 'config', {}).get("activate", {}).get("on-profile", None) + requested_section.update(section[0]) + return requested_section + if (section[0].get('deployable-name', None) == section_name): + requested_section = default_section + requested_section['deployable-name'] = section[0].get('deployable-name', None) + return None + + @staticmethod + def get_all_sections(reader: Reader, merge_on_default_section: bool = True, + update_section_data: bool = True) -> dict: + data = reader.read_config_file() + default_section = data[0][0] + sections = {"default": ParsingHelper.flatten_dict(default_section)} + is_first = True + for section in data: + if is_first: + is_first = False + continue + + current_section_name = ParsingHelper.get_current_section_name(section) + + if update_section_data: + ParsingHelper.remove_spring_section_header(section[0]) + + if current_section_name is not None: + current_section_data = copy.deepcopy(default_section) if merge_on_default_section else {} + current_section_data = merger.merge(ParsingHelper.flatten_dict(current_section_data), + copy.deepcopy(section[0])) + sections[current_section_name] = ParsingHelper.flatten_dict(current_section_data) + return sections + + @staticmethod + def remove_spring_section_header(section: dict, hack: bool = True): + if section and section.get('spring', {}).get('profiles', None) is not None and type( + section.get('spring', {}).get('profiles', None)) == str: + del section['spring']['profiles'] + + if hack: + # IMPORTANT: This is a hack to make the code work with the current schema + global deployable_section_format + deployable_section_format = "spring.profiles" + + if section and section.get('spring', {}).get('config', {}).get("activate", {}).get("on-profile", + None) is not None: + del section['spring']['config']['activate']['on-profile'] + if section and "spring~profiles" in section: + del section["spring~profiles"] + if section and "spring~config~activate~on-profile" in section: + del section["spring~config~activate~on-profile"] + + if section and section.get('deployable-name', None) is not None and type( + section.get('deployable-name', None)) == str: + del section['deployable-name'] + + if hack: + # IMPORTANT: This is a hack to make the code work with the current schema + deployable_section_format = "deployable-name" + + if section and section.get('deployable-name', None) is not None: + del section['deployable-name'] + if section and "deployable-name" in section: + del section["deployable-name"] + + @staticmethod + def get_current_section_name(section): + if section and isinstance(section, tuple) and len(section) > 0: + profiles = section[0].get('spring', {}).get('profiles', None) + if isinstance(profiles, str): + return profiles + on_profile = section[0].get('spring', {}).get('config', {}).get("activate", {}).get("on-profile", None) + if isinstance(on_profile, str): + return on_profile + if section and isinstance(section, tuple) and len(section) > 0: + return section[0].get('deployable-name', None) + return None + + @staticmethod + def flatten_dict(d, parent_key='', sep='~'): + items = [] + for k, v in d.items(): + new_key = f"{parent_key}{sep}{k}" if parent_key else k + if isinstance(v, dict): + items.extend(ParsingHelper.flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + @staticmethod + def unflatten_dict(d, separator='~'): + result_dict = {} + + for key, value in d.items(): + parts = key.split(separator) + last_d_ref = None + d_ref = result_dict + for part in parts[:-1]: + if not isinstance(d_ref, dict): + last_d_ref[".".join(parts[list(parts).index(part):])] = value + break + else: + if part not in d_ref: + d_ref[part] = {} + last_d_ref = d_ref + d_ref = d_ref[part] + + if not isinstance(d_ref, dict): + last_d_ref[".".join(parts[-2:])] = value + else: + d_ref[parts[-1]] = value + + return result_dict + + @staticmethod + def dump_properties_into_string(data: dict, priority_keys_order: list = ["spring"]): + unflatten_data = ParsingHelper.unflatten_dict(data) + dump_str = "" + for config_key in priority_keys_order: + spring_data = unflatten_data.get(config_key) + if spring_data: + dump_str = dump_str + yaml.dump({config_key: spring_data}, default_flow_style=False) + del unflatten_data[config_key] + if not unflatten_data: + return dump_str + return dump_str + yaml.dump(unflatten_data, default_flow_style=False) + + @staticmethod + def write_to_application_file(env_deployable_dictionary: dict, config_data: dict, target_dir: str, prefix: str): + # if prefix is there then adding a seperator as - + if len(prefix) > 0: + prefix = f"-{prefix}" + + for env_name in env_deployable_dictionary.keys(): + + if env_name not in config_data: + continue + + env_specific_data = config_data[env_name].copy() + + os.makedirs(target_dir, exist_ok=True) + + with open(f'{target_dir}/application{prefix}-{env_name}.yml', 'w') as application_file: + + application_file.write(ParsingHelper.dump_properties_into_string(env_specific_data["common"])) + + for deployable_name in env_deployable_dictionary.get(env_name): + + if not config_data[env_name].get(deployable_name, {}): + continue + + application_file.write(f"\n\n---\n") + + env_deployable_data = dict(config_data[env_name][deployable_name]) + ParsingHelper.remove_spring_section_header(env_deployable_data, False) + env_deployable_data[deployable_section_format] = deployable_name + application_file.write( + ParsingHelper.dump_properties_into_string(env_deployable_data)) + + @staticmethod + def write_to_application_file_with_default_section(env_deployable_dictionary: dict, env_data: dict, prefix: str): + + for env_name in env_deployable_dictionary.keys(): + + with open(f'application{prefix}-{env_name}.yml', 'w') as application_file: + + default_section_data = env_data[env_name]["default"] + + application_file.write(ParsingHelper.dump_properties_into_string(default_section_data)) + + for deployable_name in env_deployable_dictionary.get(env_name): + + if not env_data[env_name].get(deployable_name, {}) or deployable_name == "default": + continue + + application_file.write(f"\n\n---\n") + + env_deployable_data = dict(env_data[env_name][deployable_name]) + env_deployable_data[deployable_section_format] = deployable_name + application_file.write(ParsingHelper.dump_properties_into_string(env_deployable_data)) + + +def is_url(value): + """ + Check if a value is a URL using validators library. + + :param value: Value to check. + :return: True if the value is a URL, False otherwise. + """ + return validators.url(value) + + +def read_yaml_data_with_sections(file_path: str, base_data: dict = {}): + Filereader = Reader(file_path) + file_data = ParsingHelper.get_all_sections(Filereader) + + # base data mai ek section hai file data mai many + # base data dyn data + # file data static data working fine in this + + # base data static data + # file data dyn data + + copy_data = copy.deepcopy(base_data) + + for key in file_data.keys(): + if key in base_data: + base_data[key] = merger.merge(file_data[key], copy_data[key]) + else: + base_data[key] = merger.merge(file_data[key], copy_data["default"]) + + # if "default" in file_data: + # del file_data["default"] + + for key in base_data.keys(): + if key != "default": + base_data[key] = flatten_dict(merger.merge(base_data[key], base_data["default"])) + + return base_data + + +def write_schema_to_temp_file(schema_dict): + flattened_schema = flatten_dict(normalise_keys(schema_dict)) + if 'spring.profiles' in flattened_schema: + del flattened_schema['spring.profiles'] + if 'spring.config.activate.on-profile' in flattened_schema: + del flattened_schema['spring.config.activate.on-profile'] + if 'spring.application.name' in flattened_schema: + del flattened_schema['spring.application.name'] + if 'deployable-name' in flattened_schema: + del flattened_schema['deployable-name'] + + with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file: + yaml.dump(flattened_schema, temp_file) + return temp_file.name + + +def normalise_keys(d, old_char='~', new_char='.'): + """ + Recursively replace old_char with new_char in the keys of the dictionary. + + :param d: The dictionary to process. + :param old_char: The character to replace. + :param new_char: The character to replace with. + :return: A new dictionary with corrected keys. + """ + if not isinstance(d, dict): + return d + + corrected_dict = {} + for k, v in d.items(): + new_key = str(k).replace(old_char, new_char) + if isinstance(v, dict): + corrected_dict[new_key] = normalise_keys(v, old_char, new_char) + else: + corrected_dict[new_key] = v + + return corrected_dict + + +def validate_config_file_against_schema(file_path: str, schema_file: str, configs_directory_path: str, + all_key_required: bool = True): + """ + Function to validate the config file against the schema + It checks following things + 1. check consistency of the config across all the files as schema will have the structure of the config_files + 2. type error if any + 3. rule against the value + :param file_path: file path of the config file + :param schema_file: file path of the schema file + :param all_key_required: config file should have all the keys present in the schema file + :return: True if validation succeeds, False otherwise + """ + temp_schema_path = os.path.join(configs_directory_path, schema_file) + with open(temp_schema_path, 'r') as temp_file: + schema_dict = yaml.safe_load(temp_file) + temp_schema_path_to_use = write_schema_to_temp_file(schema_dict) + try: + _validators = DefaultValidators.copy() + _validators['int'] = IntOrStringInt + _validators['str'] = StringCustom + _validators['bool'] = BoolOrStringBool + _validators['json_str'] = JsonStr + _validators['num'] = NumOrStringNum + + base_data = {} + if not all_key_required: + static_file_reader = Reader(file_path.replace("-dyn", "")) + base_data = ParsingHelper.get_all_sections(static_file_reader) + else: + dyn_file_path = re.sub(r'(-[^-]+$)', r'-dyn\1', file_path) + dyn_file_reader = Reader(dyn_file_path) + base_data = ParsingHelper.get_all_sections(dyn_file_reader) + + schema = yamale.make_schema(temp_schema_path_to_use, validators=_validators) + data = read_yaml_data_with_sections(file_path, base_data) + temp_dict = copy.deepcopy(schema.dict) + schema.dict = flatten_dict(temp_dict) + final_result = True + for section_name, section_data in data.items(): + if section_name == "default": + continue + if section_data is None: + print(f"Section data is empty for {file_path} against schema {schema_file} for section {section_name}", + flush=True) + continue + flattened_data = flatten_dict(normalise_keys(section_data)) + result = schema.validate(flattened_data, section_name, strict=False) + if not result.isValid(): + print( + f"\033[91mValidation failed for environment {file_path.split('-')[-1].split('.')[0]} against schema {schema_file} for section {section_name}\033[0m", + flush=True) + print(result, flush=True) + final_result = False + + return final_result + except yamale.YamaleError as e: + print("YAML validation failed:", flush=True) + print(e.message, flush=True) + return False + + +def validate_no_cross_environment_endpoint(file_path: str, env: str) -> bool: + """ + Function to validate that the config file does not contain cross-environment references. + :param file_path: Path to the config file. + :param env: Environment name (e.g., 'prd', 'int', 'stg'). + :return: True if no cross-environment references are found, False otherwise. + """ + + status = True + + with open(file_path, 'r') as file: + try: + data = list(yaml.safe_load_all(file)) # Use safe_load_all for multiple documents + + for section_content in data: + flattened_data = flatten_dict(section_content) + for key, value in flattened_data.items(): + # Check if any invalid patterns are found in the value for the environment + invalid_patterns_for_env = invalid_patterns.get(env, []) + if isinstance(value, str) and any( + pattern in value for pattern in invalid_patterns_for_env) and not is_db_url(key, value): + print(f"\033[91mInvalid value '{value}' found in {file_path} for environment {env}\033[0m", + flush=True) + status = False + + # Check for specific zookeeper connect string patterns + if 'zookeeper.connect-string' in key or 'zookeeper~connect-string' in key or 'zookeeper.server' in key or 'zookeeper~server' in key: + elements = value.split(',') + valid_patterns = valid_zk_patterns.get(env, []) + for element in elements: + if not any(pattern in element for pattern in valid_patterns): + print( + f"\033[91mInvalid value '{value}' found in {file_path} for environment {env}\033[0m", + flush=True) + status = False + return status + except yaml.YAMLError as exc: + print(f"Error reading {file_path}: {exc}", flush=True) + return False + + +def process_config_files_of_module(configs_directory_path: str): + """ + Function to process the config files of a module + :param configs_directory_path: path where for the given modules files are residing + :return: True if all check passed else false + """ + is_invalid = False + validation_status = True + module_name = os.path.basename(configs_directory_path) + for config_file in config_files_props: + config_file_path = os.path.join(configs_directory_path, config_file['name']) + if not os.path.exists(config_file_path) and '-dev' in config_file['name']: + continue + if not os.path.exists(config_file_path): + canSkip = False + if '-int' in config_file['name']: + for whitelists in whitelists_for_preprod: + if whitelists in module_name: + canSkip = True + break + if canSkip: + continue + print(f"Validation failed for {config_file_path}", flush=True) + print(f"File {config_file_path} does not exist.", flush=True) + validation_status = False + continue + if check_for_secretes(config_file_path): + validation_status = False + + if config_file['type'] == 'static': + if not validate_config_file_against_schema(config_file_path, + config_file['schema-file-name'], + configs_directory_path): + is_invalid = True + print(f"Validation failed for {config_file_path}", flush=True) + if not validate_no_cross_environment_endpoint(config_file_path, config_file['env']): + is_invalid = True + else: + if not validate_config_file_against_schema(config_file_path, + config_file['schema-file-name'], + configs_directory_path, + False): + is_invalid = True + if is_invalid: + validation_status = False + return validation_status + + +def check_for_secretes(file_path: str): + """ + Function to check for secretes in the file + :param file_path: config file_path + :return: error_message and status of the check + """ + + if SecretHelper.is_secret_present(file_path, True): + print("\033[91mFound secrets in the config file.\033[0m", flush=True) + return True + return False + + +def find_non_matching_dicts(dict_list, dict_a): + """ + Find dictionaries in a list where specific keys do not match the given pattern. + + :param dict_list: List of tuples (dict, str). + :param dict_a: Dictionary to flatten and search for URL values. + :return: True if all keys match the pattern in the given dictionaries, False otherwise. + """ + # Step 1: Flatten dict A + flattened_dict_a = flatten_dict(dict_a) + + # Step 2: Identify keys with URL values + url_keys = [key for key, value in flattened_dict_a.items() if is_url(value)] + + # Step 3: Filter dicts in the list where keys do not match the given pattern + all_keys_match = True + for key in url_keys: + key_match_found = False + for d, s in dict_list: + if key in d and matches_pattern(d[key]): + key_match_found = True + if not key_match_found: + print(f"Key is not in proper format, name is : {key}", flush=True) + all_keys_match = False + + return all_keys_match + + +def flatten_dict(d, parent_key='', sep='.'): + """ + Flatten a nested dictionary. + + :param d: Dictionary to flatten. + :param parent_key: String to use as a prefix for keys. + :param sep: Separator for nested keys. + :return: Flattened dictionary. + """ + items = [] + for k, v in d.items(): + new_key = f'{parent_key}{sep}{k}' if parent_key else k + if isinstance(v, MutableMapping): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +def matches_pattern(value): + """ + Check if a value matches the pattern '{{stateless_service.<service-name>.<some_value>}}'. + + :param value: Value to check. + :return: True if the value matches the pattern, False otherwise. + """ + pattern = re.compile(r'\{\{(?:stateless_service|stateful_service)\.[^.]+\.[^}]+\}\}') + return bool(pattern.match(value)) + + +def clean_up(data, keys): + """ + Delete the specified nested key and clean up empty parent keys. + + :param data: The dictionary to clean up. + :param keys: A list of keys specifying the path to the nested key. + """ + if not keys: + return + + current_key = keys[0] + if len(keys) == 1: + data.pop(current_key, None) + else: + next_level = data.get(current_key, None) + if next_level is not None and isinstance(next_level, dict): + clean_up(next_level, keys[1:]) + if not next_level: + data.pop(current_key, None) + + +def deep_validate(): + configs_path = os.path.join(os.getenv("REPO_ROOT", ""), "configs") + status = True + for root, dirs, files in os.walk(configs_path): + for dir_name in dirs: + if not process_config_files_of_module(os.path.join(root, dir_name)): + status = False + return status + + +if __name__ == "__main__": + if deep_validate(): + print("Validation successful.", flush=True) + os._exit(0) + else: + print("Validation failed.", flush=True) + os._exit(1) diff --git a/resources/com/meesho/validate_configs_v2.py b/resources/com/meesho/validate_configs_v2.py new file mode 100644 index 0000000..3b8ee64 --- /dev/null +++ b/resources/com/meesho/validate_configs_v2.py @@ -0,0 +1,1261 @@ +import re +import yamale, validators +from collections.abc import MutableMapping +from deepmerge import merger +from yamale.validators import DefaultValidators, Validator +import yaml +import copy +import os +from collections import Counter +import tempfile +import json +from tabulate import tabulate + +from detect_secrets import SecretsCollection +from detect_secrets.settings import transient_settings + +from deepmerge import Merger + +merger = Merger( + [(dict, ["merge"])], + ["override"], + ["override"] +) + +deployable_section_format = "" + +invalid_patterns = { + 'stg': ['int.meesho.int', 'prd.meesho.int'], + 'int': ['stg.meesho.int'], + 'prd': ['int.meesho.int', 'stg.meesho.int'] +} +valid_zk_patterns = { +'dev': ['dev.meesho.int:2181', 'dev.meesho.int'], + 'stg': ['dev.meesho.int:2181', 'dev.meesho.int'], + 'int': ['int.meesho.int:2181', 'int.meesho.int'], + 'prd': ['config.prd.meesho.int:2181','mlp.prd.meesho.int:2181','comms.prd.meesho.int:2181', 'config.prd.meesho.int','mlp.prd.meesho.int','comms.prd.meesho.int'] +} + +whitelists_for_preprod = ['consumer', 'scheduler', 'cron'] + +db_patterns = [ + # PostgreSQL + r'^postgresql://(?:\S+):(?:\S+)@(?:\S+):\d+/\S+$', # PostgreSQL (standard) + r'^jdbc:postgresql://(?:\S+):(?:\S+)@(?:\S+):\d+/\S+$', # PostgreSQL (JDBC) + + # MySQL + r'^mysql://(?:\S+):(?:\S+)@(?:\S+):\d+/\S+$', # MySQL (standard) + r'^jdbc:mysql://*', # MySQL (JDBC) with optional query parameters + + # MongoDB + r'^mongodb://(?:\S+):(?:\S+)@(?:\S+):\d+/\S+$', # MongoDB (standard) + r'^mongodb\+srv://(?:\S+):(?:\S+)@(?:\S+)(?:/\S+)?$', # MongoDB (SRV) + + # SQLite + r'^sqlite://(?:\S+)$', # SQLite + + # Redis + r'^redis://(?:\S+):\d+$' # Redis (host:port) +] + +config_files_props = [ + { + 'name': 'application-prd.yml', + 'type': 'static', + 'schema-file-name': 'application-schema.yml', + 'env': 'prd' + }, + { + 'name': 'application-int.yml', + 'type': 'static', + 'schema-file-name': 'application-schema.yml', + 'env': 'int' + }, + { + 'name': 'application-stg.yml', + 'type': 'static', + 'schema-file-name': 'application-schema.yml', + 'env': 'stg' + }, + { + 'name': 'application-dev.yml', + 'type': 'static', + 'schema-file-name': 'application-schema.yml', + 'env': 'dev' + }, + { + 'name': 'application-dyn-prd.yml', + 'type': 'dynamic', + 'schema-file-name': 'application-schema.yml', + 'env': 'prd' + }, + { + 'name': 'application-dyn-int.yml', + 'type': 'dynamic', + 'schema-file-name': 'application-schema.yml', + 'env': 'int' + }, + { + 'name': 'application-dyn-stg.yml', + 'type': 'dynamic', + 'schema-file-name': 'application-schema.yml', + 'env': 'stg' + }, + { + 'name': 'application-dyn-dev.yml', + 'type': 'dynamic', + 'schema-file-name': 'application-schema.yml', + 'env': 'dev' + } +] + + +class NonRequiredValidator(Validator): + """ Wrapper around existing validators to make fields optional. """ + + def __init__(self, base_validator, *args, **kwargs): + super().__init__(*args, **kwargs) + self.base_validator = base_validator + + def is_valid(self, value): + """ Allow missing values by returning True when value is None """ + if value is None: + return True + return self.base_validator.is_valid(value) + + def to_python(self, value): + return self.base_validator.to_python(value) + + +class IntOrStringInt(Validator): + tag = 'int' # Custom tag to use in the schema + + def _is_valid(self, value): + # Check if value is an integer + if isinstance(value, int): + return True + # Check if value is a string that can be converted to an integer + if isinstance(value, str): + try: + int(value) + return True + except ValueError: + return False + return False + + def _format(self, value): + """Converts a valid string representation of an integer into an integer.""" + if isinstance(value, str): + try: + return int(value) + except ValueError: + pass + return value + + +class BoolOrStringBool(Validator): + tag = 'bool' + + def _is_valid(self, value): + if isinstance(value, bool): + return True + if isinstance(value, str): + value = value.lower() + if value in ['true', 'false']: + return True + return False + + def _format(self, value): + """Converts a valid string representation of a boolean into a boolean.""" + if isinstance(value, str): + value = value.lower() + if value == 'true': + return True + elif value == 'false': + return False + return value + + +class JsonStr(Validator): + tag = 'json_str' # Custom tag to use in the schema + + def _is_valid(self, value): + # Check if value is already a dict or list (already parsed JSON) + if isinstance(value, (dict, list)): + return True + # Check if value is a string that can be parsed as JSON + if isinstance(value, str): + try: + json.loads(value) + return True + except (json.JSONDecodeError, TypeError): + return False + return False + + def _format(self, value): + """Converts a valid JSON string into a parsed JSON object.""" + if isinstance(value, str): + try: + return json.loads(value) + except (json.JSONDecodeError, TypeError): + pass + return value + +class NumOrStringNum(Validator): + tag = 'num' + + def _is_valid(self, value): + if isinstance(value, (int, float)): + return True + if isinstance(value, str): + try: + float(value) # check if string can be parsed as number + return True + except ValueError: + return False + return False + + def _format(self, value): + """Converts a valid string representation of a number into int or float.""" + if isinstance(value, str): + try: + # convert to int if it's whole number, else float + f = float(value) + return int(f) if f.is_integer() else f + except ValueError: + return value + return value + +class StringCustom(Validator): + tag = 'str' + + def __init__(self, max=None, required=False): + super().__init__(required=required) + self.max_length = max + self.required = required + + def _is_valid(self, value): + errors = [] + + if self.required and value is None: + errors.append(self.fail("Value is required but missing.")) + + if not isinstance(value, (str, int, bool)): + errors.append(self.fail(f"Invalid type '{type(value).__name__}'. Expected str, int, or bool.")) + + if isinstance(value, str) and self.max_length is not None and len(value) > self.max_length: + errors.append(self.fail(f"Value exceeds maximum length of {self.max_length} characters.")) + + return errors # Return list of errors instead of True/False + + def fail(self, message): + """Override to define a custom fail message""" + return f"Validation Error: {message}" + + def validate(self, value): + """ + Override validate to return a list of validation errors. + """ + errors = self._is_valid(value) # Get errors from _is_valid + + # Validate constraints (if any exist) + for constraint in self._constraints_inst: + error = constraint.is_valid(value) + if error: + if isinstance(error, list): + errors.extend(error) + else: + errors.append(error) + + return errors # Return full list of validation errors + + def is_valid(self, value): + return not self.validate(value) # Returns True if no errors + + def _format(self, value): + if isinstance(value, int): + value = str(value) + if isinstance(value, str) and self.max_length is not None and len(value) > self.max_length: + return value[:self.max_length] + return value + + + +def is_db_url(key, value): + """ + Check if the given key-value pair is a valid database URL or name. + + Parameters: + key_value_pair (tuple): A tuple containing the key and value to check. + + Returns: + bool: True if the value is a valid database URL or name, False otherwise. + """ + + # Regex patterns for different database URL formats + db_patterns = [ + r'^(postgresql://|mysql://|sqlite://|mongodb://|oracle://|mssql://|cockroachdb://|redis://|cassandra://|neo4j://|jdbc:mysql://).+', + r'^[a-zA-Z0-9_]+$', # Just a database name (alphanumeric + underscores) + r'^[a-zA-Z0-9_]+\.([a-zA-Z0-9_]+)$' # Database name with an optional prefix (e.g., schema) + ] + + # Check if the value matches any of the patterns + for pattern in db_patterns: + if re.match(pattern, value): + return True + + return False + + +class Reader: + """ + This class provides methods to read the yml configuration file which has sections. + File structure should be as follows: + + default_Data: + + --- + spring: + profiles: sections_1 + sections_1_data + + --- + spring: + profiles: sections_2 + sections_2_data + + --- + spring: + profiles: sections_3 + sections_3_data + """ + + def __init__(self, config_file_path: str): + self.config_file_path = config_file_path + + def read_config_file(self): + """ + Read the config file and return the data in the form of dictionary. + """ + data = yamale.make_data(self.config_file_path) + return data + + # Function to process each YAML file + def read_without_sections(self): + with open(self.config_file_path, 'r') as file: + try: + data = yaml.safe_load(file) + return data + except yaml.YAMLError as exc: + print(f"Error reading {self.config_file_path}: {exc}", flush=True) + return None + + +class DictUtils: + @staticmethod + def sub_dict(keys: list[str], data: dict) -> dict: + """ + Function to extract the dictionary from the keys + :param keys: keys to extract + :param data: dictionary of data + :return: dictionary with the keys + """ + return {key: data[key] for key in keys} + + @staticmethod + def clean_up(data, keys): + """ + Delete the specified nested key and clean up empty parent keys. + + :param data: The dictionary to clean up. + :param keys: A list of keys specifying the path to the nested key. + """ + if not keys: + return + + current_key = keys[0] + if len(keys) == 1: + data.pop(current_key, None) + else: + next_level = data.get(current_key, None) + if next_level is not None and isinstance(next_level, dict): + DictUtils.clean_up(next_level, keys[1:]) + if not next_level: + data.pop(current_key, None) + + @staticmethod + def remove_common_keys(dict_of_dicts, common_data): + for key in dict_of_dicts: + for common_key in common_data.keys(): + if common_key in dict_of_dicts[key]: + del dict_of_dicts[key][common_key] + return dict_of_dicts + + @staticmethod + def find_common_values(dict_of_dicts): + dict_list = list(dict_of_dicts.values()) + common_values = dict_list[0].copy() + for d in dict_list[1:]: + common_values = {k: v for k, v in common_values.items() if k in d and d[k] == v} + return common_values + + @staticmethod + def remove_keys(dictionary: dict, keys: list): + for key in keys: + if key in dictionary: + del dictionary[key] + + @staticmethod + def extract_dict_with_chosen_keys(dictionary: dict, keys: list): + new_dict = dict() + for key in keys: + if key in dictionary: + new_dict[key] = dictionary[key] + return new_dict + + @staticmethod + def extract_common_data_outside(nested_dict: dict): + common_values = DictUtils.find_common_values(nested_dict) + dict_list = DictUtils.remove_common_keys(nested_dict, common_values) + # Assuming common_values and dict_list are defined + merged_dict = common_values.copy() # Start with keys and values from common_values + merged_dict.update(dict_list) # Adds keys and values from each dictionary in dict_list to merged_dict + return merged_dict + + @staticmethod + def extract_common_majority(dict_of_dicts): + dict_keys = list(dict_of_dicts.keys()) + + if not dict_keys: + return dict_of_dicts + + # Step 1: Identify common keys across all inner dictionaries + common_keys = set(dict_of_dicts[dict_keys[0]].keys()) + for key in dict_keys[1:]: + common_keys.intersection_update(dict_of_dicts[key].keys()) + + common_dict = {} + + for key in common_keys: + if key in ["spring~application~name", "spring~profiles", "spring~config~activate~on-profile","deployable-name"]: + continue + serialized_values = [json.dumps(dict_of_dicts[d][key], sort_keys=True) for d in dict_keys] + most_common_serialized_value, count = Counter(serialized_values).most_common(1)[0] + most_common_value = json.loads(most_common_serialized_value) + common_dict[key] = most_common_value + for d in set(dict_keys): + if json.dumps(dict_of_dicts[d][key], sort_keys=True) == most_common_serialized_value: + del dict_of_dicts[d][key] + dict_of_dicts["common"] = common_dict + return dict_of_dicts + + @staticmethod + def extract_common(dict_of_dicts, common_data: dict): + """ + Extract common data from the dictionary of dictionaries and if some keys are there which are common in + dict_of_dicts but not present in common_data then for them we will use the majority value + :param dict_of_dicts: dict: deployable -> config_key -> config_value + :param common_data: dict: config_key -> config_value + :return: + """ + for key, value in common_data.items(): + for deployable in dict_of_dicts.keys(): + if key in dict_of_dicts[deployable] and dict_of_dicts[deployable][key] == value: + dict_of_dicts[deployable].pop(key) + dict_of_dicts = DictUtils.extract_common_majority(dict_of_dicts) + dict_of_dicts["common"] = merger.merge(common_data, dict_of_dicts.get("common", {})) + return dict_of_dicts + + +class JsonFileUtils: + + @staticmethod + def get_keys_by_line_numbers(file_path: str, line_numbers: list[int]) -> list[str]: + keys = [] + with open(file_path, 'r') as file: + for i, line in enumerate(file, start=1): + if i in line_numbers: + # Extract the key from the line + key = line.strip().split(':')[0].replace('"', '').strip() + keys.append(key) + return keys + + @staticmethod + def write_dict_to_json_file(data: dict, file_path: str): + with open(file_path, "w") as json_file: + json.dump(data, json_file, indent=4) + + +class SecretFinder: + + def __init__(self, file_path: str, is_exclude_keyword_detector: bool = False): + self.file_path = file_path + self.secrets = SecretsCollection() + self.settings = { + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "OpenAIDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TelegramBotTokenDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ] + } + if not is_exclude_keyword_detector: + self.settings["plugins_used"].append({ + "name": "KeywordDetector", + "keyword_exclude": "" + } + ) + + def fetch_secrets(self) -> dict: + """ + Function to check for secretes in the filex + :return: line numbers of the secretes in the file + """ + with transient_settings(self.settings): + self.secrets.scan_file(self.file_path) + + return self.secrets.json().get(self.file_path, {}) + + +class SecretHelper: + def __init__(self): + pass + + @staticmethod + def extract_secrets_positions(secrets: dict) -> list[int]: + """ + Function to extract the line numbers of the secrets + :param secrets: secret find result from the secret finder + :return: line numbers of the secretes in the file + """ + line_numbers = set() + if len(secrets) > 0: + for secret in secrets: + line_numbers.add(secret['line_number']) + return sorted(list(line_numbers)) + + @staticmethod + def is_secret_present(file_path: str, is_exclude_keyword_detector: bool = False) -> bool: + """ + Function to check for secretes in the file + :param is_exclude_keyword_detector: if we want to exclude keyword detector + :param file_path: config file_path + :return: error_message and status of the check + """ + secrets = SecretFinder(file_path, is_exclude_keyword_detector).fetch_secrets() + if len(secrets) != 0: + print(json.dumps(secrets, indent=2), flush=True) + secrets_line_numbers = SecretHelper.extract_secrets_positions(secrets) + return len(secrets_line_numbers) > 0 + + @staticmethod + def fetch_secrets_keys_from_json(file_path: str) -> list[str]: + """ + Function to fetch the secret keys from the json file + :return: list of secret keys + """ + secrets = SecretFinder(file_path).fetch_secrets() + secrets_line_numbers = SecretHelper.extract_secrets_positions(secrets) + secret_keys = JsonFileUtils.get_keys_by_line_numbers(file_path, secrets_line_numbers) + return secret_keys + + @staticmethod + def filter_secrets_from_dict(data: dict, secret_keys: list[str]) -> dict: + """ + Function to filter the secrets from the dictionary + :param secret_keys: secret keys + :param data: dictionary of data + :return: dictionary without the secrets + """ + return DictUtils.sub_dict(secret_keys, data) + + @staticmethod + def find_secrets_in_json_files(directory_path: str) -> dict: + + file_to_secrets_dict = dict() + + for filename in os.listdir(directory_path): + if filename.endswith(".json"): + file_path = os.path.join(directory_path, filename) + secrets = SecretHelper.fetch_secrets_keys_from_json(file_path) + file_to_secrets_dict[filename] = secrets + + return file_to_secrets_dict + + +class ParsingHelper: + + @staticmethod + def get_default_section(reader: Reader): + return reader.read_config_file()[0][0] + + @staticmethod + def get_section_data(reader: Reader, section_name: str) -> dict: + data = reader.read_config_file() + default_section = data[0][0] + is_first = True + for section in data: + if is_first: + is_first = False + continue + if (section[0].get('spring', {}).get('profiles', None) == section_name or + section[0].get('spring', {}).get('config', {}).get("activate", {}).get("on-profile", + None) == section_name): + requested_section = default_section + requested_section['spring']['profiles'] = section[0].get('spring', {}).get('profiles', None) + requested_section['spring']['config']['activate']['on-profile'] = section[0].get('spring', {}).get( + 'config', {}).get("activate", {}).get("on-profile", None) + requested_section.update(section[0]) + return requested_section + if (section[0].get('deployable-name', None) == section_name): + requested_section = default_section + requested_section['deployable-name'] = section[0].get('deployable-name', None) + return None + + @staticmethod + def get_all_sections(reader: Reader, merge_on_default_section: bool = True, + update_section_data: bool = True) -> dict: + data = reader.read_config_file() + default_section = data[0][0] + sections = {"default": ParsingHelper.flatten_dict(default_section)} + is_first = True + for section in data: + if is_first: + is_first = False + continue + + current_section_name = ParsingHelper.get_current_section_name(section) + + if current_section_name is not None: + current_section_data = copy.deepcopy(default_section) if merge_on_default_section else {} + current_section_data = merger.merge(ParsingHelper.flatten_dict(current_section_data), + copy.deepcopy(section[0])) + sections[current_section_name] = ParsingHelper.flatten_dict(current_section_data) + return sections + + @staticmethod + def remove_spring_section_header(section: dict, hack: bool = True): + if section and section.get('spring', {}).get('profiles', None) is not None and type( + section.get('spring', {}).get('profiles', None)) == str: + del section['spring']['profiles'] + + if hack: + # IMPORTANT: This is a hack to make the code work with the current schema + global deployable_section_format + deployable_section_format = "spring.profiles" + + if section and section.get('spring', {}).get('config', {}).get("activate", {}).get("on-profile", + None) is not None: + del section['spring']['config']['activate']['on-profile'] + if section and "spring~profiles" in section: + del section["spring~profiles"] + if section and "spring~config~activate~on-profile" in section: + del section["spring~config~activate~on-profile"] + + if section and section.get('deployable-name', None) is not None and type( + section.get('deployable-name', None)) == str: + del section['deployable-name'] + + if hack: + # IMPORTANT: This is a hack to make the code work with the current schema + deployable_section_format = "deployable-name" + + if section and section.get('deployable-name', None) is not None: + del section['deployable-name'] + if section and "deployable-name" in section: + del section["deployable-name"] + + @staticmethod + def get_current_section_name(section): + if section and isinstance(section, tuple) and len(section) > 0: + profiles = section[0].get('spring', {}).get('profiles', None) + if isinstance(profiles, str): + return profiles + on_profile = section[0].get('spring', {}).get('config', {}).get("activate", {}).get("on-profile", None) + if isinstance(on_profile, str): + return on_profile + if section and isinstance(section, tuple) and len(section) > 0: + return section[0].get('deployable-name', None) + return None + + @staticmethod + def flatten_dict(d, parent_key='', sep='~'): + items = [] + for k, v in d.items(): + new_key = f"{parent_key}{sep}{k}" if parent_key else k + if isinstance(v, dict): + items.extend(ParsingHelper.flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + @staticmethod + def unflatten_dict(d, separator='~'): + result_dict = {} + + for key, value in d.items(): + parts = key.split(separator) + last_d_ref = None + d_ref = result_dict + for part in parts[:-1]: + if not isinstance(d_ref, dict): + last_d_ref[".".join(parts[list(parts).index(part):])] = value + break + else: + if part not in d_ref: + d_ref[part] = {} + last_d_ref = d_ref + d_ref = d_ref[part] + + if not isinstance(d_ref, dict): + last_d_ref[".".join(parts[-2:])] = value + else: + d_ref[parts[-1]] = value + + return result_dict + + @staticmethod + def dump_properties_into_string(data: dict, priority_keys_order: list = ["spring"]): + unflatten_data = ParsingHelper.unflatten_dict(data) + dump_str = "" + for config_key in priority_keys_order: + spring_data = unflatten_data.get(config_key) + if spring_data: + dump_str = dump_str + yaml.dump({config_key: spring_data}, default_flow_style=False) + del unflatten_data[config_key] + if not unflatten_data: + return dump_str + return dump_str + yaml.dump(unflatten_data, default_flow_style=False) + + @staticmethod + def write_to_application_file(env_deployable_dictionary: dict, config_data: dict, target_dir: str, prefix: str): + # if prefix is there then adding a seperator as - + if len(prefix) > 0: + prefix = f"-{prefix}" + + for env_name in env_deployable_dictionary.keys(): + + if env_name not in config_data: + continue + + env_specific_data = config_data[env_name].copy() + + os.makedirs(target_dir, exist_ok=True) + + with open(f'{target_dir}/application{prefix}-{env_name}.yml', 'w') as application_file: + + application_file.write(ParsingHelper.dump_properties_into_string(env_specific_data["common"])) + + for deployable_name in env_deployable_dictionary.get(env_name): + + if not config_data[env_name].get(deployable_name, {}): + continue + + application_file.write(f"\n\n---\n") + + env_deployable_data = dict(config_data[env_name][deployable_name]) + ParsingHelper.remove_spring_section_header(env_deployable_data, False) + env_deployable_data[deployable_section_format] = deployable_name + application_file.write( + ParsingHelper.dump_properties_into_string(env_deployable_data)) + + @staticmethod + def write_to_application_file_with_default_section(env_deployable_dictionary: dict, env_data: dict, prefix: str): + + for env_name in env_deployable_dictionary.keys(): + + with open(f'application{prefix}-{env_name}.yml', 'w') as application_file: + + default_section_data = env_data[env_name]["default"] + + application_file.write(ParsingHelper.dump_properties_into_string(default_section_data)) + + for deployable_name in env_deployable_dictionary.get(env_name): + + if not env_data[env_name].get(deployable_name, {}) or deployable_name == "default": + continue + + application_file.write(f"\n\n---\n") + + env_deployable_data = dict(env_data[env_name][deployable_name]) + env_deployable_data[deployable_section_format] = deployable_name + application_file.write(ParsingHelper.dump_properties_into_string(env_deployable_data)) + + +def is_url(value): + """ + Check if a value is a URL using validators library. + + :param value: Value to check. + :return: True if the value is a URL, False otherwise. + """ + return validators.url(value) + + +def read_yaml_data_with_sections(file_path: str, base_data: dict = {}): + Filereader = Reader(file_path) + file_data = ParsingHelper.get_all_sections(Filereader) + + # base data mai ek section hai file data mai many + # base data dyn data + # file data static data working fine in this + + # base data static data + # file data dyn data + + copy_data = copy.deepcopy(base_data) + + for key in file_data.keys(): + if key in base_data: + base_data[key] = merger.merge(file_data[key], copy_data[key]) + else: + base_data[key] = merger.merge(file_data[key], copy_data["default"]) + + # if "default" in file_data: + # del file_data["default"] + + for key in base_data.keys(): + if key != "default": + base_data[key] = flatten_dict(merger.merge(base_data[key], base_data["default"])) + + return base_data + + +def write_schema_to_temp_file(schema_dict): + flattened_schema = flatten_dict(normalise_keys(schema_dict)) + + with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file: + yaml.dump(flattened_schema, temp_file) + return temp_file.name + + +def normalise_keys(d, old_char='~', new_char='.'): + """ + Recursively replace old_char with new_char in the keys of the dictionary. + + :param d: The dictionary to process. + :param old_char: The character to replace. + :param new_char: The character to replace with. + :return: A new dictionary with corrected keys. + """ + if not isinstance(d, dict): + return d + + corrected_dict = {} + for k, v in d.items(): + new_key = str(k).replace(old_char, new_char) + if isinstance(v, dict): + corrected_dict[new_key] = normalise_keys(v, old_char, new_char) + else: + corrected_dict[new_key] = v + + return corrected_dict + + +def validate_config_file_against_schema(file_path: str, schema_file: str, configs_directory_path: str, + all_key_required: bool = True): + """ + Function to validate the config file against the schema + It checks following things + 1. check consistency of the config across all the files as schema will have the structure of the config_files + 2. type error if any + 3. rule against the value + :param file_path: file path of the config file + :param schema_file: file path of the schema file + :param all_key_required: config file should have all the keys present in the schema file + :return: True if validation succeeds, False otherwise + """ + final_result= True + temp_schema_path = os.path.join(configs_directory_path, schema_file) + with open(temp_schema_path, 'r') as temp_file: + schema_dict = yaml.safe_load(temp_file) + temp_schema_path_to_use = write_schema_to_temp_file(schema_dict) + try: + _validators = DefaultValidators.copy() + _validators['int'] = IntOrStringInt + _validators['str'] = StringCustom + _validators['bool'] = BoolOrStringBool + _validators['json_str'] = JsonStr + _validators['num'] = NumOrStringNum + + base_data = {} + if not all_key_required: + static_file_reader = Reader(file_path.replace("-dyn", "")) + base_data = ParsingHelper.get_all_sections(static_file_reader) + else: + dyn_file_path = re.sub(r'(-[^-]+$)', r'-dyn\1', file_path) + dyn_file_reader = Reader(dyn_file_path) + base_data = ParsingHelper.get_all_sections(dyn_file_reader) + + schema = yamale.make_schema(temp_schema_path_to_use, validators=_validators) + data = read_yaml_data_with_sections(file_path, base_data) + temp_dict = copy.deepcopy(schema.dict) + schema.dict = flatten_dict(temp_dict) + error_rows = [] + for section_name, section_data in data.items(): + if section_name == "default": + continue + if section_data is None: + error_rows.append([ + color_env(file_path.split('-')[-1].split('.')[0]), # Environment + section_name, + "", + f"Section data is empty for {file_path} against schema {schema_file} for section {section_name}" + ]) + continue + flattened_data = flatten_dict(normalise_keys(section_data)) + result = schema.validate(flattened_data, section_name, strict=True) + if not result.isValid(): + for err in result.errors: + # Try to extract the key and error message + # Example error: "spring.cloud.zookeeper.config.enabled: Unexpected element" + if ':' in err: + key, issue = err.split(':', 1) + else: + key, issue = "", err + error_rows.append([ + color_env(file_path.split('-')[-1].split('.')[0]), # Environment + section_name, + key.strip(), + issue.strip() + ]) + final_result = False + + if error_rows: + print("\n" + tabulate( + error_rows, + headers=["Environment", "Deployable Name", "Key", "Error/Issue"], + tablefmt="fancy_grid" + ) + "\n", flush=True) + return final_result + except yamale.YamaleError as e: + print("YAML validation failed:", flush=True) + print(e.message, flush=True) + return False + + +def validate_no_cross_environment_endpoint(file_path: str, env: str) -> bool: + """ + Function to validate that the config file does not contain cross-environment references. + :param file_path: Path to the config file. + :param env: Environment name (e.g., 'prd', 'int', 'stg'). + :return: True if no cross-environment references are found, False otherwise. + """ + status = True + error_rows = [] + + with open(file_path, 'r') as file: + try: + data = list(yaml.safe_load_all(file)) # Use safe_load_all for multiple documents + + for section_content in data: + flattened_data = flatten_dict(section_content) + for key, value in flattened_data.items(): + # Check if any invalid patterns are found in the value for the environment + invalid_patterns_for_env = invalid_patterns.get(env, []) + if isinstance(value, str) and any( + pattern in value for pattern in invalid_patterns_for_env) and not is_db_url(key, value): + error_rows.append([ + color_env(env), + key, + value, + "Invalid cross-environment value" + ]) + status = False + + # Check for specific zookeeper connect string patterns + if 'zookeeper.connect-string' in key or 'zookeeper~connect-string' in key or 'zookeeper.server' in key or 'zookeeper~server' in key: + elements = value.split(',') + valid_patterns = valid_zk_patterns.get(env, []) + for element in elements: + if not any(pattern in element for pattern in valid_patterns): + error_rows.append([ + color_env(env), + key, + element, + "Invalid zookeeper connect string" + ]) + status = False + + if error_rows: + print("\n" + tabulate( + error_rows, + headers=["Environment", "Key", "Value", "Error/Issue"], + tablefmt="fancy_grid" + ) + "\n", flush=True) + return status + except yaml.YAMLError as exc: + print(f"Error reading {file_path}: {exc}", flush=True) + return False + + +def process_config_files_of_module(configs_directory_path: str): + """ + Function to process the config files of a module + :param configs_directory_path: path where for the given modules files are residing + :return: True if all check passed else false + """ + is_invalid = False + validation_status = True + module_name = os.path.basename(configs_directory_path) + for config_file in config_files_props: + config_file_path = os.path.join(configs_directory_path, config_file['name']) + if not os.path.exists(config_file_path) and '-dev' in config_file['name']: + continue + if not os.path.exists(config_file_path): + canSkip = False + if '-int' in config_file['name']: + for whitelists in whitelists_for_preprod: + if whitelists in module_name: + canSkip = True + break + if canSkip: + continue + print(f"Validation failed for {config_file_path}", flush=True) + print(f"File {config_file_path} does not exist.", flush=True) + validation_status = False + continue + if check_for_secretes(config_file_path): + validation_status = False + + if config_file['type'] == 'static': + if not validate_config_file_against_schema(config_file_path, + config_file['schema-file-name'], + configs_directory_path): + is_invalid = True + if not validate_no_cross_environment_endpoint(config_file_path, config_file['env']): + is_invalid = True + else: + if not validate_config_file_against_schema(config_file_path, + config_file['schema-file-name'], + configs_directory_path, + False): + is_invalid = True + if is_invalid: + validation_status = False + return validation_status + + +def check_for_secretes(file_path: str): + """ + Function to check for secretes in the file + :param file_path: config file_path + :return: error_message and status of the check + """ + + if SecretHelper.is_secret_present(file_path, True): + print("\033[91mFound secrets in the config file.\033[0m", flush=True) + return True + return False + + +def find_non_matching_dicts(dict_list, dict_a): + """ + Find dictionaries in a list where specific keys do not match the given pattern. + + :param dict_list: List of tuples (dict, str). + :param dict_a: Dictionary to flatten and search for URL values. + :return: True if all keys match the pattern in the given dictionaries, False otherwise. + """ + # Step 1: Flatten dict A + flattened_dict_a = flatten_dict(dict_a) + + # Step 2: Identify keys with URL values + url_keys = [key for key, value in flattened_dict_a.items() if is_url(value)] + + # Step 3: Filter dicts in the list where keys do not match the given pattern + all_keys_match = True + for key in url_keys: + key_match_found = False + for d, s in dict_list: + if key in d and matches_pattern(d[key]): + key_match_found = True + if not key_match_found: + print(f"Key is not in proper format, name is : {key}", flush=True) + all_keys_match = False + + return all_keys_match + + +def flatten_dict(d, parent_key='', sep='.'): + """ + Flatten a nested dictionary. + + :param d: Dictionary to flatten. + :param parent_key: String to use as a prefix for keys. + :param sep: Separator for nested keys. + :return: Flattened dictionary. + """ + items = [] + for k, v in d.items(): + new_key = f'{parent_key}{sep}{k}' if parent_key else k + if isinstance(v, MutableMapping): + items.extend(flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +def matches_pattern(value): + """ + Check if a value matches the pattern '{{stateless_service.<service-name>.<some_value>}}'. + + :param value: Value to check. + :return: True if the value matches the pattern, False otherwise. + """ + pattern = re.compile(r'\{\{(?:stateless_service|stateful_service)\.[^.]+\.[^}]+\}\}') + return bool(pattern.match(value)) + + +def clean_up(data, keys): + """ + Delete the specified nested key and clean up empty parent keys. + + :param data: The dictionary to clean up. + :param keys: A list of keys specifying the path to the nested key. + """ + if not keys: + return + + current_key = keys[0] + if len(keys) == 1: + data.pop(current_key, None) + else: + next_level = data.get(current_key, None) + if next_level is not None and isinstance(next_level, dict): + clean_up(next_level, keys[1:]) + if not next_level: + data.pop(current_key, None) + + +def deep_validate(): + configs_path = os.path.join(os.getenv("REPO_ROOT", ""), "configs") + status = True + if not os.path.exists(configs_path): + return True + for root, dirs, files in os.walk(configs_path): + for dir_name in dirs: + if not process_config_files_of_module(os.path.join(root, dir_name)): + status = False + return status + + +def color_env(env): + color_map = { + "dev": "\033[94m", # Blue + "stg": "\033[93m", # Yellow + "int": "\033[92m", # Green + "prd": "\033[91m", # Red + } + reset = "\033[0m" + return f"{color_map.get(env, '')}{env}{reset}" + + +if __name__ == "__main__": + if deep_validate(): + print("Validation successful.", flush=True) + os._exit(0) + else: + print("Validation failed.", flush=True) + os._exit(1) diff --git a/resources/com/meesho/values.yaml b/resources/com/meesho/values.yaml new file mode 100644 index 0000000..a07f21f --- /dev/null +++ b/resources/com/meesho/values.yaml @@ -0,0 +1,202 @@ +<% if (deploymentStrategy == 'canary'){ print "canary:\n enabled: true\n slackChannel: ${canary.slackChannel}\n enableManualPromotion: ${enableManualPromotion}\n skipAnalysis: ${canary.skipAnalysis}\n service:\n port: 80\n targetPort: ${primary_port}\n" ; print " progressDeadlineSeconds: ${canary.progressDeadlineSeconds}\n minCanaryReplicas: ${minCanaryReplicas}\n maxCanaryReplicas: ${maxCanaryReplicas}\n analysisInterval: ${canary.analysisInterval}\n analysisThreshold: ${canary.analysisThreshold}\n analysisMaxWeight: ${canary.analysisMaxWeight}\n analysisStepWeight: ${canary.analysisStepWeight}\n analysisMetrics:\n thresholdRangeMin: ${canary.analysisMetrics.thresholdRangeMin}\n interval: ${canary.analysisMetrics.interval}\n" } else { print "canary:\n enabled: false\n" } %>repoName: ${repo_name} +cron: + enabled: false + serviceAccount: + enabled: false +applicationName: ${app_name} +autoscaling: + enabled: ${as_enabled} + maxReplicas: ${as_max} + minReplicas: ${as_min} + pollingInterval: ${as_poll} + scaledown: + policies: + - periodseconds: ${as_down_period} + type: Pods + value: ${as_down_pod_count} + selectpolicy: Min + stabilizationWindowSeconds: ${as_down_stable_window} + scaleup: + policies: + - periodseconds: ${as_up_period} + type: Pods + value: ${as_up_pod_count} + - periodseconds: ${as_up_period} + type: Percent + value: ${as_up_pod_percentage} + selectpolicy: Max + stabilizationWindowSeconds: ${as_up_stable_window} +<% if(!triggers){print " triggers:\n - metadata:\n value: \"${as_trigger_value}\"\n metricType: ${as_trigger_type}\n type: ${as_trigger_metric}"} else {println ' triggers:'; for(val in triggers){if(val instanceof Map){ val.each{k,v -> if (v instanceof Map) { println ' - '+k+':'; v.each{a,b -> if(b.isNumber()){println " ${a}: \"${b}\"" } else {println " ${a}: ${b}" }}} else {println" ${k}: ${v}"}}}}} %> +replicaCount: ${replica_count} +otel_enabled: ${otel_enabled} +metrics_mode: ${metrics_mode} +deployment: + affinity: {} + podDistributionSkew: ${podDistributionSkew} + args: +<% for (arg in deployment_args){ if (arg.contains("Xms") || arg.contains("Xmx")) {println ''} else {println ' - '+arg}} %> + <% if (appConfigEnabled) { %>- -Dspring.profiles.active=${environment},${app_name},dyn-${environment} <% } %> + <% if (appConfigEnabled) { %>- -Dspring.config.additional-location=/opt/config/application-${environment}.yml,/opt/config/application-dyn-${environment}.yml<% } %> + - -Xms${xms} + - -Xmx${xmx} + - -XX:ActiveProcessorCount=${activeProcessorCount} + - -javaagent:/opt/jmx_exporter.jar=8880:/jmx/jmx-config.yaml + <% if (telegraf_metrics) { %>- -Dtelegraf-metrics-enabled=true<% } else { %>- -Dtelegraf-metrics-enabled=false<% } %> + <% if (otel_enabled) { %> + - -javaagent:/opt/opentelemetry-javaagent.jar + - -Dotel.exporter.otlp.protocol=grpc + - -Dotel.logs.exporter=none + - -Dotel.metrics.exporter=none + - -Dotel.traces.exporter=otlp + - -Dotel.traces.sampler=traceidratio + <% if (environment != "stg") { %> + - -Dotel.traces.sampler.arg=${otel_traces_sampler_arg} + <% } %> + - -Dotel.resource.attributes=service.name=${app_name},application.name=${app_name},api.name=${app_name},cx.application.name=${app_name},cx.subsystem.name=${app_name} + <% } %> + - -jar + - <% print module=='module_less'?repo_name:module %>.jar + command: + - java + enabled: true + env: + - name: PRISMSDK_ENVIRONMENT + value: ${prismsdk_environment} + envFrom: + secretRef: ${app_name} + image: + pullPolicy: IfNotPresent + pullSecret: "" + repository: ${registry}/${environment}/${build_team}/<% print module=='module_less'?repo_name.toLowerCase():repo_name.toLowerCase()+'/'+module %> + tag: ${tag} +<% if(lifecycle) { println ' lifecycle:\n preStop:\n exec:\n command:'; for(val in lifecycle.preStop.exec.command){ println " - $val" }} else {print " lifecycle:\n preStop:\n exec:\n command:\n - /bin/bash\n - -c\n - kill -SIGQUIT 1 ; /bin/sleep 120\n "} %> + minReadySeconds: 10 + podAnnotations: + jmx.io/path: /metrics + jmx.io/port: "8880" + jmx.io/scrape: "true" + <% if (nodeSelector.contains("arm64") && (environment=="int" || environment=="prd")) { print 'telegraf.influxdata.com/image: 847438129436.dkr.ecr.ap-southeast-1.amazonaws.com/telegraf:1.24.4-arm64' } %> + <% if (appMetrics) { print 'prometheus.io/path: /actuator/prometheus' %> + <% print 'prometheus.io/port: "'+app_port+'"' %> + <% print 'prometheus.io/scrape: "true"'} %> +<% if (pod_annotations) {pod_annotations.each{k,v -> if(v instanceof String) { println " ${k}: '${v}'"} else { println " ${k}: ${v}" }};} else {print ''} %> + ports: + - containerPort: ${app_port} + name: http + protocol: TCP + - containerPort: 8880 + name: metric + protocol: TCP + probes: + liveness: + failureThreshold: ${liveness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${liveness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${liveness_success_threshold} + timeoutSeconds: ${liveness_timeout_seconds} + readiness: + failureThreshold: ${readiness_failure_threshold} + initialDelaySeconds: ${initialDelaySeconds} + path: ${health_check} + periodSeconds: ${readiness_period_seconds} + port: http + scheme: HTTP + successThreshold: ${readiness_success_threshold} + timeoutSeconds: ${readiness_timeout_seconds} + resources: + limits: + cpu: ${cpu_limit} + memory: ${memory_limit}i + requests: + cpu: ${cpu_request} + memory: ${memory_request}i + revisionHistoryLimit: 6 +<% if(serviceAccount){println ' serviceAccount:\n annotations:';serviceAccount.annotations.each{k,v -> println " ${k}: ${v}"};println " enabled: ${serviceAccount.enabled}"} else {print ' serviceAccount:\n annotations: null\n enabled: false'} %> + nodeSelector: + ${nodeSelector}: ${nodeSelectorValue} +<% if (hostAliases){println ' hostAliases:';for(arr in hostAliases){println ' - ip: '+arr.get("ip"); println ' hostnames:'; for (ele in arr.get("hostnames")) { println ' - '+ele } }} %> + tolerations: + - effect: NoSchedule + key: ${nodeSelector} + operator: Equal + value: ${nodeSelectorValue} + updateStrategy: + strategy: +<% if (deploymentStrategy == 'recreate') { print " type: Recreate"} else { print " type: RollingUpdate\n rollingUpdate:\n maxUnavailable: 0%\n maxSurge: ${maxSurge}%" } %> +externalSecret: + annotations: +<% if (external_secrets_annotations) {external_secrets_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true + path: meesho/${vault_env}/${bu}/${team}/${app_name} + version: ${tag} + name: ${env_ns}-${app_name} + target: ${app_name} +fullnameOverride: "" +ingress: + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "false" + nginx.ingress.kubernetes.io/ssl-redirect: "false" +<% if (ingress_annotations) {ingress_annotations.each{k,v -> println " ${k}: ${v}"};} else {print ''} %> + enabled: true +<% if(grpc_host){println " grpc_hosts:\n - host: ${grpc_host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else if (grpc_hosts) {println " grpc_hosts:";for(host_arr in grpc_hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> +<% if(host){println " hosts:\n - host: ${host}\n paths:\n - pathType: ImplementationSpecific\n path: /"} else {println " hosts:";for(host_arr in hosts){println " - host: ${host_arr.host}\n paths:";for(path_arr in host_arr.paths){println " - pathType: ${path_arr.pathType}\n path: ${path_arr.path}"; if(path_arr.targetService) { println " targetService: ${path_arr.targetService}" } }}} %> ingressClassName: ${ingress_class} + servicePort: http + enableWebsocket: ${enableWebsocket} + slowStart: + enabled: <% if (slowStartWindow) { print "true" } else { print "false" } %> + window: <% if (slowStartWindow) { print "${slowStartWindow}" } else { print "120s" } %> + aggression: <% if (slowStartAggression) { print "${slowStartAggression}" } else { print "1.0" } %> + minPercent: <% if (slowStartMinPercent) { print "${slowStartMinPercent}" } else { print "10" } %> +jmxconfig: + enabled: true +labels: + priority: <% print priority?:'p1' %> + priority_v2: <% print priority_v2?:'cp3' %> + primary_owner: ${primary_owner} + secondary_owner: ${secondary_owner} + env: ${environment_norm} + team: ${team_norm} + bu: ${bu_norm} + <% if (service_type_norm) { println "service_type: ${service_type_norm}" } %> + commit_id: ${commit_id} +nameOverride: "" +namespace: ${env_ns}-${app_name} +podDisruptionBudget: + enabled: <% if (pdbMaxUnavailable) { print "true" } else { print "false" } %> + maxUnavailable: ${pdbMaxUnavailable} + minAvailable: ${pdbMinAvailable} +podSecurityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsUser: 65534 +service: +<% if (service_annotations) { println " annotations:"; service_annotations.each{k,v -> println " ${k}: \"${v}\""};} else { println " annotations: null" } %> + enabled: true +<% if (addon_ports) { for (p in addon_ports) { println " addons:"; println " - name: ${p.name}"; println " targetPort: ${p.targetPort}"; println " type: ${p.type}"; }} else { println " addon_ports: []" } %> + ports: + - name: http + port: 80 + protocol: TCP + targetPort: ${primary_port} + type: ClusterIP + +createContourGateway: <% if (createContourGateway) { print "${createContourGateway}" } else { print "false" } %> +contourResponseTimeout: ${contourResponseTimeout} + + +appConfig: + enabled: <% if(appConfigEnabled) { print "true" } else { print "false"} %> + env: ${environment} +<% if(appConfigEnabled) {%> + staticAppConfig: + data: | +${staticAppConfigData.trim().replaceAll("(?m)^", " ")} + dynamicAppConfig: + data: | +${dynamicAppConfigData.trim().replaceAll("(?m)^", " ")} +<% } +%> + diff --git a/resources/com/meesho/values_properties.yaml b/resources/com/meesho/values_properties.yaml new file mode 100644 index 0000000..ede22c1 --- /dev/null +++ b/resources/com/meesho/values_properties.yaml @@ -0,0 +1,25 @@ +--- + +as_poll: ${as_poll} +as_down_period: ${as_down_period} +as_up_period: ${as_up_period} +as_up_stable_window: ${as_up_stable_window} +ingress_class: ${ingress_class} +host: ${host} +hostAliases: false +nodeSelector: <% if(arch == 'arm64'){print environment=='dev' || environment=='ftr' ? bu+'-'+arch: environment == 'int' ? team+"-"+arch:nodeSelector } else { print environment == 'dev' || environment == 'ftr' ? bu: environment == 'int' ? bu+"-int" : nodeSelector} %> +triggers: + - metadata: + value: "${as_trigger_value}" + metricType: ${as_trigger_type} + type: ${as_trigger_metric} +canary: + progressDeadlineSeconds: 300 + analysisInterval: 120s + analysisThreshold: 5 + analysisMaxWeight: 5 + analysisStepWeight: 5 + analysisMetrics: + thresholdRangeMin: 0.99 + interval: 1m + skipAnalysis: false \ No newline at end of file diff --git a/resources/org/meesho/dev-pod.yaml b/resources/org/meesho/dev-pod.yaml new file mode 100644 index 0000000..7c84c89 --- /dev/null +++ b/resources/org/meesho/dev-pod.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + bu: "infra" + team: "devops" + service: "jenkins-dev" + env: "dev" + priority: "p0" + type: "jenkins" + component: "jenkins-agent" +spec: + serviceAccountName: jenkins-dev-agent + containers: + - name: devops-tools + image: asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/build-tools:lunar-v2.0.22 + imagePullPolicy: Always + resources: + requests: + memory: "6G" + cpu: "2" + limits: + memory: "1000G" + cpu: "100" + volumeMounts: + - mountPath: "/root" + name: "cache" + readOnly: false + env: + - name: TZ + value: Asia/Kolkata + - name: DOCKER_HOST + value: dind-dev-svc + command: + - cat + tty: true + nodeSelector: + dedicated: jenkins + tolerations: + - key: "dedicated" + operator: "Equal" + value: "jenkins" + effect: "NoSchedule" + volumes: + - name: cache + persistentVolumeClaim: + claimName: pvc-jenkins-agents-cache-dev diff --git a/resources/org/meesho/prd-pod.yaml b/resources/org/meesho/prd-pod.yaml new file mode 100644 index 0000000..379e5bf --- /dev/null +++ b/resources/org/meesho/prd-pod.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + bu: "infra" + team: "devops" + service: "jenkins-prd" + env: "prd" + priority: "p0" + type: "jenkins" + component: "jenkins-agent" +spec: + serviceAccountName: jenkins-prd-agent + containers: + - name: devops-tools + + image: asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/build-tools:lunar-v2.0.22 + + imagePullPolicy: Always + resources: + requests: + memory: "6G" + cpu: "2" + limits: + memory: "1000G" + cpu: "100" + volumeMounts: + - mountPath: "/root" + name: "cache" + readOnly: false + env: + - name: TZ + value: Asia/Kolkata + - name: DOCKER_HOST + value: dind-prd-svc + command: + - cat + tty: true + nodeSelector: + dedicated: jenkins + tolerations: + - key: "dedicated" + operator: "Equal" + value: "jenkins" + effect: "NoSchedule" + volumes: + - name: cache + persistentVolumeClaim: + claimName: pvc-jenkins-agents-cache-prd-new diff --git a/resources/org/meesho/prd-sidecar-pod.yaml b/resources/org/meesho/prd-sidecar-pod.yaml new file mode 100644 index 0000000..6a37ca2 --- /dev/null +++ b/resources/org/meesho/prd-sidecar-pod.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + bu: "infra" + team: "devops" + service: "jenkins-prd" + env: "prd" + priority: "p0" + type: "jenkins" + component: "jenkins-agent" +spec: + serviceAccountName: jenkins-prd-agent + containers: + - name: devops-tools + image: asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/build-tools:lunar-v2.0.21 + imagePullPolicy: Always + resources: + requests: + memory: "6G" + cpu: "2" + limits: + memory: "1000G" + cpu: "100" + volumeMounts: + - mountPath: "/root" + name: "cache" + readOnly: false + env: + - name: TZ + value: Asia/Kolkata + - name: DOCKER_HOST + value: dind-prd-svc + command: + - cat + tty: true + + - name: dind + image: asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/docker:28-dind + securityContext: + privileged: true + env: + - name: DOCKER_TLS_CERTDIR + value: "" + resources: + requests: + cpu: "1" + memory: "2Gi" + limits: + cpu: "1" + memory: "2Gi" + volumeMounts: + - name: dind-storage + mountPath: /var/lib/docker + nodeSelector: + dedicated: jenkins + tolerations: + - key: "dedicated" + operator: "Equal" + value: "jenkins" + effect: "NoSchedule" + volumes: + - name: cache + persistentVolumeClaim: + claimName: pvc-jenkins-agents-cache-prd-new + - name: dind-storage + emptyDir: {} + diff --git a/resources/org/meesho/stg-pod.yaml b/resources/org/meesho/stg-pod.yaml new file mode 100644 index 0000000..85bbd4a --- /dev/null +++ b/resources/org/meesho/stg-pod.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + bu: "infra" + team: "devops" + service: "jenkins-stg" + env: "stg" + priority: "p0" + type: "jenkins" + component: "jenkins-agent" +spec: + serviceAccountName: jenkins-dev-agent + containers: + - name: devops-tools + image: asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/build-tools:lunar-v2.0.22 + imagePullPolicy: Always + resources: + requests: + memory: "6G" + cpu: "2" + limits: + memory: "1000G" + cpu: "100" + volumeMounts: + - mountPath: "/root" + name: "cache" + readOnly: false + env: + - name: TZ + value: Asia/Kolkata + - name: DOCKER_HOST + value: dind-dev-new-svc.jenkins-new.svc.cluster.local + command: + - cat + tty: true + + nodeSelector: + dedicated: jenkins + tolerations: + - key: "dedicated" + operator: "Equal" + value: "jenkins" + effect: "NoSchedule" + volumes: + - name: cache + persistentVolumeClaim: + claimName: pvc-jenkins-agents-cache-dev diff --git a/resources/org/meesho/stg-sidecar-pod.yaml b/resources/org/meesho/stg-sidecar-pod.yaml new file mode 100644 index 0000000..31c8a33 --- /dev/null +++ b/resources/org/meesho/stg-sidecar-pod.yaml @@ -0,0 +1,67 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + bu: "infra" + team: "devops" + service: "jenkins-stg" + env: "stg" + priority: "p0" + type: "jenkins" + component: "jenkins-agent" +spec: + serviceAccountName: jenkins-dev-agent + containers: + - name: devops-tools + image: asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/build-tools:lunar-v2.0.21 + imagePullPolicy: Always + resources: + requests: + memory: "6G" + cpu: "2" + limits: + memory: "1000G" + cpu: "100" + volumeMounts: + - mountPath: "/root" + name: "cache" + readOnly: false + env: + - name: TZ + value: Asia/Kolkata + - name: DOCKER_HOST + value: dind-dev-new-svc.jenkins-new.svc.cluster.local + command: + - cat + tty: true + + - name: dind + image: asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/docker:28-dind + securityContext: + privileged: true + env: + - name: DOCKER_TLS_CERTDIR + value: "" # Disable TLS so we can connect via HTTP on port 2375 + resources: + requests: + cpu: "1" + memory: "2Gi" + limits: + cpu: "1" + memory: "2Gi" + volumeMounts: + - name: dind-storage + mountPath: /var/lib/docker + nodeSelector: + dedicated: jenkins + tolerations: + - key: "dedicated" + operator: "Equal" + value: "jenkins" + effect: "NoSchedule" + volumes: + - name: cache + persistentVolumeClaim: + claimName: pvc-jenkins-agents-cache-dev + - name: dind-storage + emptyDir: {} diff --git a/resources/org/meesho/templates/maven-3.3-jdk-8.sh b/resources/org/meesho/templates/maven-3.3-jdk-8.sh new file mode 100644 index 0000000..3e0ebfd --- /dev/null +++ b/resources/org/meesho/templates/maven-3.3-jdk-8.sh @@ -0,0 +1,84 @@ +#!/bin/bash +prepareDockerfileWithBuild(){ +cat << EOF > Dockerfile-${artifactId} +# This sample, non-production-ready template describes an Amazon EC2 instance and an Elastic Load Balancer. +# © 2020 Amazon Web Services, Inc. or its affiliates. All Rights Reserved. +# This AWS Content is provided subject to the terms of the AWS Customer Agreement available at +# http://aws.amazon.com/agreement or other written agreement between Customer and either +# Amazon Web Services, Inc. or Amazon Web Services EMEA SARL or both. +FROM 766380763301.dkr.ecr.ap-south-1.amazonaws.com/build/maven:3.3-jdk-8 as BUILD +WORKDIR /usr/src/app +COPY . /usr/src/app +RUN mvn -s sandbox-settings.xml clean install -DskipTests +RUN mkdir -p /var/log/${artifactId} && touch /var/log/${artifactId}/gc.log +FROM 766380763301.dkr.ecr.ap-south-1.amazonaws.com/build/java:8-jdk-slim-secure_v1.0 +ADD https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar /opt/jmx_exporter.jar +### Config added through configmap +# COPY config.yaml /opt/config.yaml +EXPOSE 8880 8010 +COPY --from=BUILD /usr/src/app/target/${artifactId}-${version}.jar /opt/target/${artifactId}.jar +COPY --from=BUILD /var/log/${artifactId} /var/log/${artifactId} +WORKDIR /opt/target +CMD ["${artifactId}.jar", "-javaagent:/opt/jmx_exporter.jar=8880:/opt/config/jmx-config.yaml", \ + "-XX:MinRAMPercentage=50.0", "-XX:MaxRAMPercentage=80.0", \ + "-XX:+UseParallelGC -XX:+PrintGCDateStamps -XX:+PrintGCDetails", \ + "-XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime", "-XX:+PrintHeapAtGC", \ + "-Xloggc:/var/log/${artifactId}/gc.log", \ + "-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=9000k", \ + "-Xms2G", "-Xmx2G"] +EOF + +} + +prepareDockerfileNoBuild(){ +cat << EOF > Dockerfile-${artifactId} +# This sample, non-production-ready template describes an Amazon EC2 instance and an Elastic Load Balancer. +# © 2020 Amazon Web Services, Inc. or its affiliates. All Rights Reserved. +# This AWS Content is provided subject to the terms of the AWS Customer Agreement available at +# http://aws.amazon.com/agreement or other written agreement between Customer and either +# Amazon Web Services, Inc. or Amazon Web Services EMEA SARL or both. +FROM 766380763301.dkr.ecr.ap-south-1.amazonaws.com/build/maven:3.3-jdk-8 as BUILD +#FROM asia-southeast1-docker.pkg.dev/supply-poc-351106/meesho-devops/maven:3.3-jdk-8 as BUILD +RUN mkdir -p /var/log/${artifactId} && touch /var/log/${artifactId}/gc.log +FROM 766380763301.dkr.ecr.ap-south-1.amazonaws.com/build/java:8-jdk-slim-secure_v1.0 +#FROM asia-southeast1-docker.pkg.dev/supply-poc-351106/meesho-devops/java:8 +ADD https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.15.0/jmx_prometheus_javaagent-0.15.0.jar /opt/jmx_exporter.jar +### Config added through configmap +# COPY config.yaml /opt/config.yaml +EXPOSE 8880 8010 +COPY ${artifactId}/target/*.jar /opt/target/${artifactId}.jar +COPY --from=BUILD /var/log/${artifactId} /var/log/${artifactId} +WORKDIR /opt/target +CMD ["${artifactId}.jar", "-javaagent:/opt/jmx_exporter.jar=8880:/opt/config/jmx-config.yaml", \ + "-XX:MinRAMPercentage=50.0", "-XX:MaxRAMPercentage=80.0", \ + "-XX:+UseParallelGC -XX:+PrintGCDateStamps -XX:+PrintGCDetails", \ + "-XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime", "-XX:+PrintHeapAtGC", \ + "-Xloggc:/var/log/${artifactId}/gc.log", \ + "-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=9000k", \ + "-Xms2G", "-Xmx2G"] +EOF + +} + +main(){ + if [[ $# == 4 ]];then + export artifactId=$1 + export version=$2 + export doBuild=$3 + export repoName=$4 + if [[ $doBuild == "YES" ]];then + rm -f Dockerfile-${artifactId} + prepareDockerfileWithBuild + else + rm -f Dockerfile-${artifactId} + prepareDockerfileNoBuild + fi + ls -latr + cat Dockerfile-${artifactId} + else + echo "Please provide required parameters" + exit 1 + fi +} + +main $@ \ No newline at end of file diff --git a/resources/org/meesho/templates/node-12.22.sh b/resources/org/meesho/templates/node-12.22.sh new file mode 100644 index 0000000..5b4499e --- /dev/null +++ b/resources/org/meesho/templates/node-12.22.sh @@ -0,0 +1,50 @@ +#!/bin/bash +prepareDockerfile(){ +cat << EOF > Dockerfile-${artifactId} + +ARG ACCOUNT_ID=766380763301 + +FROM \${ACCOUNT_ID}.dkr.ecr.ap-southeast-1.amazonaws.com/build/node:12.22.1-slim as build-env + +WORKDIR /usr/src/app + +COPY package*.json ./ + +RUN npm install + +FROM \${ACCOUNT_ID}.dkr.ecr.ap-southeast-1.amazonaws.com/build/node:12.22.1-alpine + +# for health checks +# RUN apk add --update --no-cache curl=7.74.0-r1 + +USER node:1000 + +COPY --chown=node:1000 --from=build-env /usr/src/app /app + +WORKDIR /app + +EXPOSE 2020 + +COPY --chown=node:1000 . . + +CMD ["npm", "start"] + +EOF + +} + +main(){ + if [[ $# == 1 ]];then + export artifactId=$1 + rm -f Dockerfile-${artifactId} + prepareDockerfile + ls -latr + cat Dockerfile-${artifactId} + else + echo "Please provide required parameters" + exit 1 + fi +} + +main $@ + diff --git a/resources/org/meesho/toolchain-pod.yaml b/resources/org/meesho/toolchain-pod.yaml new file mode 100644 index 0000000..0e8d375 --- /dev/null +++ b/resources/org/meesho/toolchain-pod.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + bu: "infra" + team: "toolchain" + service: "jenkins-toolchain" + env: "stg" + priority: "p0" + type: "jenkins" + component: "jenkins-agent" +spec: + serviceAccountName: jenkins-toolchain-dev-agent + containers: + - name: devops-tools + image: asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/build-tools:lunar-v2.0.22 + imagePullPolicy: Always + resources: + requests: + memory: "6G" + cpu: "2" + limits: + memory: "1000G" + cpu: "100" + volumeMounts: + - mountPath: "/root" + name: "cache" + readOnly: false + env: + - name: TZ + value: Asia/Kolkata + - name: DOCKER_HOST + value: toolchain-dind-dev-svc.jenkins-toolchain.svc.cluster.local + command: + - cat + tty: true + nodeSelector: + dedicated: toolchain-jenkins + tolerations: + - key: "dedicated" + operator: "Equal" + value: "toolchain-jenkins" + effect: "NoSchedule" + volumes: + - name: cache + persistentVolumeClaim: + claimName: pvc-toolchain-jenkins-agents-cache-dev diff --git a/review-learnings.md b/review-learnings.md new file mode 100644 index 0000000..51166b8 --- /dev/null +++ b/review-learnings.md @@ -0,0 +1,614 @@ +# PR Review Guide + +> Derived from human review discussions across 26 PRs merged into `main` (PRs #306–#727). +> Focus: P0 (production-impacting) and P1 (bugs, build reliability, architecture, error handling, observability) signals. +> Excluded: Style, formatting, refactors, lint-level feedback. + +--- + +## P0 — Must Not Miss (Production Impact) + +```yaml +- id: P0_DEBUG_URL_IN_PRODUCTION_CONFIG + severity: P0 + title: Debug/tunnel URLs must not reach production pipeline code + trigger_paths: + - "src/com/meesho/stages/*.groovy" + - "src/com/meesho/utilities/*.groovy" + required_evidence: + - "No lhr.life, ngrok, localtunnel, or similar tunnel hostnames present in any switch/case or URL assignment" + - "All cicdBaseUrl values point to stable internal DNS names (e.g. turbo-turtle.meeshogcp.in)" + suggested_fix: + - "Revert any lhr.life or debug tunnel URL to the canonical internal DNS before merging" + - "Add a pre-commit or CI grep that fails on *.lhr.life, *.ngrok.io, *.tunnel.* patterns in .groovy files" + why: > + PRs #596 and #664 both accidentally merged lhr.life tunnel URLs into deployRingmaster.groovy, + replacing the production Turbo-Turtle endpoint. This would break all production and int-environment + deployments silently — pipelines would attempt callbacks to an ephemeral tunnel that no longer exists. + references: + - "PR #596" + - "PR #664" + +- id: P0_SENSITIVE_DEBUG_PRINT_IN_PIPELINE + severity: P0 + title: Pipeline code must not print SSH private keys or credentials to console + trigger_paths: + - "src/com/meesho/utilities/addSSHKey.groovy" + - "src/com/meesho/**/*.groovy" + - "resources/com/meesho/*Dockerfile" + required_evidence: + - "No `cat`, `echo`, or `print` of private key file contents (id_github_jenkins, id_rsa, etc.)" + - "No `ls -al` of credential file paths in shipped pipeline code" + suggested_fix: + - "Remove any `cat ./id_github_jenkins` or `ls -al <key-path>` lines before merging" + - "Debug prints of credentials are acceptable only in local branches; must be stripped before PR" + why: > + PR #634 shipped `cat ./id_github_jenkins` inside addSSHKey.groovy. Jenkins build logs are accessible + to all users with job read access — printing private key content exposes the key to anyone who can + view the console output, which is a critical credential leak. + references: + - "PR #634" + +- id: P0_HARDCODED_IP_IN_PIPELINE + severity: P0 + title: Internal service endpoints must use DNS names, not hardcoded IPs + trigger_paths: + - "src/com/meesho/stages/*.groovy" + - "src/com/meesho/utilities/*.groovy" + required_evidence: + - "No bare IP addresses (e.g. 172.x.x.x, 10.x.x.x) used as curl/HTTP targets in pipeline code (known exception: securityScan.groovy:11 uses 172.31.5.29:63232 pending remediation)" + - "All internal service calls use DNS-resolvable hostnames" + suggested_fix: + - "Replace IP literals with DNS names before merging" + - "If no DNS exists yet, create one before the PR lands — do not ship with the IP as a placeholder" + why: > + PR #681 hardcoded `http://172.23.72.116:5002` as the build-callback endpoint in notify.groovy. + IP addresses break silently when infra is rebalanced or services are migrated, with no indication + in the pipeline code that the endpoint has moved. + references: + - "PR #681" + +- id: P0_CONFIG_TEMPLATE_INJECTION_SAFETY + severity: P0 + title: User-supplied config data embedded in YAML templates must be sanitized + trigger_paths: + - "resources/com/meesho/values.yaml" + - "src/com/meesho/stages/deployArgoCD.groovy" + required_evidence: + - "Any `.trim().replaceAll(...)` applied to externally-sourced config data is reviewed for correctness" + - "Indentation injection via `replaceAll(\"(?m)^\", \" \")` is validated against multiline edge cases" + - "User-controlled strings containing YAML special characters (`:`, `|`, `>`, `#`) are escaped or block-quoted" + suggested_fix: + - "Apply strict input validation or schema-based parsing before interpolating user config into YAML templates" + - "Prefer a YAML library for construction over string interpolation" + why: > + PR #343 reviewer flagged that 'people can put strange stuff in their configs' when raw config data + is string-interpolated into the values.yaml template. Malformed or adversarial input can corrupt + the YAML structure silently, causing ArgoCD deployments to use wrong or empty config. + references: + - "PR #343" +``` + +--- + +## P1 — Important Improvements (Reliability, Build Health, Architecture) + +```yaml +- id: P1_EXTERNAL_BINARY_DOWNLOAD_IN_BUILD + severity: P1 + title: External binaries must not be downloaded on every pipeline run + trigger_paths: + - "src/com/meesho/stages/buildGo.groovy" + - "src/com/meesho/stages/buildMaven.groovy" + - "src/com/meesho/stages/buildNode.groovy" + - "resources/com/meesho/*Dockerfile" + required_evidence: + - "No `curl -LO https://binaries.sonarsource.com/...` or `curl -LO https://go.dev/dl/...` inside the stage body" + - "Sonar-scanner binary is baked into the relevant Docker image or fetched from internal Artifactory" + - "Go toolchain is fetched from Artifactory (not go.dev) when a non-image version is needed" + suggested_fix: + - "Bake sonar-scanner into the builder base image; reference it as a path, not a curl download" + - "For Go version management, pull tarballs from internal Artifactory instead of go.dev to avoid external dependency and improve speed" + - "If dynamic version selection is needed, cache the download to a Jenkins agent workspace layer" + why: > + PR #643 added `curl -LO https://binaries.sonarsource.com/.../sonar-scanner-cli-5.0.1.3006-linux.zip` + and `curl -LO https://go.dev/dl/go${goVersion}.linux-amd64.tar.gz` inside the build stage body. + This increases build time on every run, creates a hard dependency on external internet connectivity, + and makes pipelines brittle when upstream URLs change or are rate-limited. + references: + - "PR #643" + +- id: P1_MISSING_PREREQUISITE_FILE_CHECK + severity: P1 + title: Required config files must be verified to exist before stages that depend on them + trigger_paths: + - "src/com/meesho/stages/buildGo.groovy" + - "src/com/meesho/stages/buildMaven.groovy" + - "src/com/meesho/stages/buildNode.groovy" + required_evidence: + - "If a stage uses `sonar-project.properties`, `pom.xml`, or equivalent config, existence is checked before the stage runs" + - "Missing file results in a clear failure message, not a cryptic scanner error mid-stage" + suggested_fix: + - "Add `if (!fileExists('sonar-project.properties')) { error('sonar-project.properties not found — cannot run Sonar scan') }` before the scan stage" + why: > + PR #643 reviewer flagged that if sonar-project.properties does not exist, the scan stage fails with + a cryptic error mid-execution. An upfront existence check surfaces the real problem immediately and + avoids partial stage execution that leaves the pipeline in an ambiguous state. + references: + - "PR #643" + +- id: P1_TEST_FAILURE_MUST_FAIL_PIPELINE + severity: P1 + title: Test failures must cause the build to fail, not silently continue + trigger_paths: + - "src/com/meesho/stages/buildGo.groovy" + - "src/com/meesho/stages/buildMaven.groovy" + - "src/com/meesho/stages/buildNode.groovy" + required_evidence: + - "Non-zero test exit codes propagate as build failures or are explicitly documented as intentional with a tracking ticket" + - "Any `|| true` or equivalent suppression on a test command has a comment explaining why" + suggested_fix: + - "Remove `if (testExitCode != 0) { sh('echo tests failed, pipeline continues') }` patterns" + - "If tests are intentionally non-blocking, capture the result as a warning in the build report rather than silently absorbing the failure" + why: > + PR #643 had a pattern where Go test failures were caught, logged as 'pipeline will continue', and + execution proceeded to Sonar and Docker stages. This allows broken code to be packaged and deployed + while appearing to pass CI, defeating the purpose of test gating. + references: + - "PR #643" + +- id: P1_CENTRALIZED_CLEANUP_IN_FINALLY + severity: P1 + title: Build cleanup logic must be in a centralized finally block, not scattered try-catches + trigger_paths: + - "src/com/meesho/stages/buildMaven.groovy" + - "src/com/meesho/stages/buildNode.groovy" + - "src/com/meesho/stages/buildGo.groovy" + - "src/com/meesho/stages/buildRust.groovy" + required_evidence: + - "New failure/cleanup paths are handled via a shared cleanup function called from the build's finally block" + - "Cleanup is not added only to the specific branch/case that was fixed — it covers all known failure scenarios" + suggested_fix: + - "Extract a `cleanupOnFailure(config)` function in each build stage file" + - "Call it from the `finally` block of the top-level try/catch in buildMaven, buildNode, buildGo, buildRust" + why: > + PR #533 reviewer requested that cleanup on failed Docker tag operations be moved to the overall + `finally` block rather than an ad-hoc inner try-catch. Scattered cleanup means new failure modes + (aborts, timeouts, new stages) bypass cleanup silently — a centralized `finally` ensures cleanup + is language-agnostic and future-proof. + references: + - "PR #533" + +- id: P1_SEMANTIC_FLAG_REUSE + severity: P1 + title: Pipeline control flags must have precise semantics; do not reuse existing flags for new meanings + trigger_paths: + - "src/com/meesho/stages/buildGo.groovy" + - "src/com/meesho/stages/buildMaven.groovy" + - "src/com/meesho/stages/buildNode.groovy" + - "src/com/meesho/stages/hotFix.groovy" + required_evidence: + - "New pipeline behaviors controlled by a new, clearly named env flag (not piggybacking on `hot_fix`, `skip_sonar`, etc.)" + - "The flag name matches the intent: `revert_pr`, `skip_quality_gate`, `is_config_only_change`" + suggested_fix: + - "Introduce `env.revert_pr` rather than overloading `env.hot_fix` for revert PR detection" + - "Document new flags in the devops-lib README or pipeline configuration guide" + why: > + PR #578 used `env.hot_fix` to skip quality gates for revert PRs, which a reviewer flagged as a + semantic hack. PR #495 added a redundant `env.hot_fix || skip_sonar` check when `skip_sonar` was + already sufficient. Overloading flags creates confusion about which paths are active and makes + future changes to either flag risky. + references: + - "PR #578" + - "PR #495" + +- id: P1_DOCKER_LAYER_CACHING_DEPENDENCY_FIRST + severity: P1 + title: Dockerfiles must copy dependency manifests before source to enable layer caching + trigger_paths: + - "resources/com/meesho/*Dockerfile" + - "resources/com/meesho/rust-Dockerfile" + required_evidence: + - "Dependency manifest files (Cargo.toml/Cargo.lock, package.json/package-lock.yaml, go.mod/go.sum, pom.xml) are COPY'd and dependencies fetched before COPY . ." + - "Source-only changes do not invalidate the dependency install layer" + suggested_fix: + - "For Rust: `COPY Cargo.toml Cargo.lock ./` → `RUN cargo fetch` → `COPY . .` → `RUN cargo build --release`" + - "For Node: `COPY package.json package-lock.yaml ./` → `RUN npm ci` → `COPY . .` → `RUN npm run build`" + why: > + PR #650 reviewer flagged that the Rust Dockerfile did `COPY . .` before fetching dependencies, + meaning every source change triggers a full `cargo build` including all dependency compilation. + Splitting the copy invalidates the dependency layer only on manifest changes, dramatically reducing + incremental build times. + references: + - "PR #650" + +- id: P1_HARDCODED_TOOL_VERSION_IN_DOCKERFILE + severity: P1 + title: Base image tool versions must be developer-configurable, not hardcoded in Dockerfiles + trigger_paths: + - "resources/com/meesho/*Dockerfile" + - "resources/com/meesho/rust-Dockerfile" + required_evidence: + - "Runtime/language version (Rust, Go, Java, Node) is parameterized as a build ARG or resolved from the service config, not hardcoded" + - "Pattern is consistent with how Go, Java, and Python versions are handled in existing Dockerfiles" + suggested_fix: + - "Use `ARG RUST_VERSION=1.86.0` and `FROM .../rust:${RUST_VERSION} AS builder` — pass the version from the pipeline config" + why: > + PR #650 hardcoded `FROM .../rust:1.86.0` in the Rust Dockerfile. Reviewer noted this is inconsistent + with Go, Java, and Python where the version is developer-configurable via the service config. Services + needing a different Rust version would require a devops-lib PR rather than a self-service config change. + references: + - "PR #650" + +- id: P1_DYNAMIC_DOCKERFILE_MUTATION_VIA_SHELL + severity: P1 + title: Dockerfiles must not be constructed by appending lines via shell echo at build time + trigger_paths: + - "src/com/meesho/stages/buildNode.groovy" + - "src/com/meesho/stages/buildGo.groovy" + - "src/com/meesho/stages/buildMaven.groovy" + required_evidence: + - "No `sh \"echo '...' >> Dockerfile-${artifactId}\"` patterns in build stage code" + - "Conditional Dockerfile content is handled via ARG/build-arg, multi-stage targets, or separate Dockerfile templates" + suggested_fix: + - "Use `ARG USE_CAC_PATH` in the Dockerfile template and conditional `RUN` blocks based on that arg" + - "If the feature is truly mode-specific, maintain a separate Dockerfile template (e.g., `node-cac-Dockerfile`) rather than mutating a shared one at runtime" + why: > + PR #707 appended `RUN truncate -s 0 .env` to the Dockerfile via `sh echo >>` when CAC was enabled. + Runtime Dockerfile mutation makes the actual image instructions invisible in code review, untestable + in isolation, and fragile to ordering assumptions. + references: + - "PR #707" + +- id: P1_FEATURE_BRANCH_IN_ENVIRONMENT_MAP + severity: P1 + title: Feature branch names must not be hardcoded in the environment/branch mapping tables + trigger_paths: + - "src/com/meesho/utilities/constructParam.groovy" + - "src/com/meesho/stages/deployArgoCD.groovy" + required_evidence: + - "The `environment_map` and `branch_param_map` in constructParam.groovy / deployArgoCD.groovy contain only permanent branch names (master, main, develop, gcp-main, etc.)" + - "No `feature/...` branch names are present in these maps" + suggested_fix: + - "Remove test-only feature branch entries before merging" + - "If special branch routing is needed for integration testing, use a mechanism that is not committed to the shared library (e.g., a per-repo override config)" + why: > + PR #707 added `feature/config_as_code_main` to both environment_map and branch_param_map as a + testing convenience. Reviewer questioned the purpose and the author confirmed it was test-only. + Leaving it in the shared library would mean all repos inheriting devops-lib would attempt to route + this branch to staging, which is incorrect. + references: + - "PR #707" + +- id: P1_SCRIPT_REUSE_OVER_DUPLICATION + severity: P1 + title: Config validation scripts must be shared, not duplicated per language + trigger_paths: + - "resources/com/meesho/validate_configs*.py" + required_evidence: + - "New language-specific validation needs extend `validate_configs_v2.py` (or the current canonical script) rather than creating a new file" + - "Duplication is justified only when the new script has fundamentally different logic that cannot be parameterized" + suggested_fix: + - "Extend `validate_configs_v2.py` with language-specific config schemas rather than copying it to `validate_configs_node.py`" + - "Use a `--lang` argument or config-driven schema selection to handle per-language differences" + why: > + PR #707 introduced `validate_configs_node.py` (470 lines) when `validate_configs_v2.py` already + existed. Duplicating the validation script means bug fixes and new validation rules must be applied + in multiple places, and the scripts will drift over time. + references: + - "PR #707" + +- id: P1_TOOLS_IN_BASE_IMAGE_NOT_BUILD_STEP + severity: P1 + title: Globally-used tools must be baked into the base image, not installed per build + trigger_paths: + - "resources/com/meesho/*Dockerfile" + - "src/com/meesho/stages/*.groovy" + required_evidence: + - "No `npm install -g <tool>` or `apt-get install <tool>` inside the Docker build RUN steps that are also used globally" + - "Tools like sonar-scanner, pnpm, and librdkafka are in the base image where they apply to all builds" + suggested_fix: + - "Move `npm install -g sonar-scanner` and similar global installs to the base Node image build process" + - "Cut a new base image version that includes the tool rather than installing it on each application build" + why: > + PR #306 flagged `npm install -g sonar-scanner` in the application Dockerfile — this runs on every + build for every service. Moving it to the base image eliminates the per-build install cost and + ensures all services use the same tool version. + references: + - "PR #306" + +- id: P1_DOWNLOADED_BINARY_HASH_VERIFICATION + severity: P1 + title: Externally downloaded binaries must have their integrity verified + trigger_paths: + - "src/com/meesho/stages/buildGo.groovy" + - "resources/com/meesho/*Dockerfile" + required_evidence: + - "Any `curl -LO` downloading a binary is followed by a hash/checksum verification step" + - "Or the download is replaced by an Artifactory-hosted artifact with controlled provenance" + suggested_fix: + - "After downloading, run `sha256sum -c <expected-hash>` or use the official checksum file provided by the vendor" + - "Prefer Artifactory-hosted binaries (Meesho-controlled) to eliminate the need for runtime checksum verification" + why: > + PR #643 reviewer requested: 'Could we also verify that the downloaded artifact matches its computed hash?' + Pipelines downloading binaries from the internet without verification are vulnerable to supply-chain + attacks if the upstream URL is compromised or served a modified artifact. + references: + - "PR #643" + +- id: P1_WRONG_ENV_VAR_IN_CONTAINER + severity: P1 + title: Container environment variables must be appropriate for the runtime language + trigger_paths: + - "resources/com/meesho/rust-values.yaml" + - "resources/com/meesho/rust-Dockerfile" + - "resources/com/meesho/*-values.yaml" + required_evidence: + - "GOMAXPROCS is not set in non-Go containers (Rust, Java, Node, Python)" + - "Any env vars carried over from a template are validated as relevant to the target runtime" + suggested_fix: + - "Remove `GOMAXPROCS` from rust-values.yaml — it is a Go runtime tuning parameter and has no effect in a Rust binary" + - "When creating a new language template by copying an existing one, audit all env vars for relevance" + why: > + PR #650 set `GOMAXPROCS` in the Rust service values.yaml template, which a reviewer questioned. + This is a Go-specific knob that controls goroutine scheduling and is meaningless in a Rust process. + Beyond being confusing, copying env vars blindly from templates creates operational noise and + incorrect assumptions for future operators. + references: + - "PR #650" + +- id: P1_SHARED_CONSTANTS_FOR_REGISTRY_PATHS + severity: P1 + title: Repeated infrastructure constants (registry URLs, base paths) must be defined as shared variables + trigger_paths: + - "src/com/meesho/stages/*.groovy" + - "src/com/meesho/utilities/*.groovy" + required_evidence: + - "Registry base URLs like `asia-southeast1-docker.pkg.dev/meesho-central-dev-0622/toolchain` appear in a single shared constant, not repeated inline across stage files" + - "New stage files reference the shared constant rather than re-declaring the string" + suggested_fix: + - "Define `TOOLCHAIN_REGISTRY_BASE` (or equivalent) in a shared constants file/class in `src/com/meesho/utilities/`" + - "Reference it in notify.groovy and any other stage that constructs toolchain image paths" + why: > + PR #681 reviewer flagged `def base_registry = \"asia-southeast1-docker.pkg.dev/meesho-central-dev-0622/toolchain\"` + as an inline constant that should be a shared variable. Duplicating registry paths means a project + or region migration requires finding and updating every occurrence individually. + references: + - "PR #681" +``` + +--- + +## Common Failure Patterns Observed + +```yaml +- id: PATTERN_DEBUG_ARTIFACT_LEFT_IN_PRODUCTION_CODE + title: Debug artifacts and test configurations merged into main pipeline code + description: > + Multiple PRs merged temporary debug configurations (tunnel URLs, hardcoded IPs, feature branch + mappings, SSH key prints, debug echo statements) that were introduced during local testing and + not cleaned up before merge. The pattern appears in deployRingmaster.groovy (lhr.life URLs), + notify.groovy (IP address), constructParam.groovy (feature branch mapping), and addSSHKey.groovy + (credential printing). The shared library nature of devops-lib means these reach all services + immediately on merge. + related_rules: + - "P0_DEBUG_URL_IN_PRODUCTION_CONFIG" + - "P0_SENSITIVE_DEBUG_PRINT_IN_PIPELINE" + - "P0_HARDCODED_IP_IN_PIPELINE" + - "P1_FEATURE_BRANCH_IN_ENVIRONMENT_MAP" + references: + - "PR #596" + - "PR #634" + - "PR #664" + - "PR #681" + - "PR #707" + +- id: PATTERN_PER_BUILD_TOOL_INSTALLATION + title: Tools that should be in base images are installed on every build run + description: > + Several PRs introduced per-build installation of tools that belong in the builder base image: + sonar-scanner downloaded via curl on every Go and Node build, Go toolchain downloaded from go.dev + on every Sonar scan, librdkafka and pnpm installed in application Dockerfiles. This pattern slows + all builds, adds external network dependencies to the critical path, and creates version drift + when different builds download different patch versions. + related_rules: + - "P1_EXTERNAL_BINARY_DOWNLOAD_IN_BUILD" + - "P1_TOOLS_IN_BASE_IMAGE_NOT_BUILD_STEP" + - "P1_DOWNLOADED_BINARY_HASH_VERIFICATION" + references: + - "PR #306" + - "PR #643" + - "PR #650" + +- id: PATTERN_COPY_PASTE_WITHOUT_AUDIT + title: New language support created by copying existing templates without auditing all fields + description: > + When adding Rust CICD support (PR #650) and Node CAC support (PR #707), reviewers flagged that + values.yaml templates, Dockerfiles, and validation scripts were copied from an existing language + without removing inapplicable settings (GOMAXPROCS in Rust) or reusing shared logic + (validate_configs_node.py vs validate_configs_v2.py). Copy-paste propagates bugs and misconfigurations + from the source template to the new language. + related_rules: + - "P1_WRONG_ENV_VAR_IN_CONTAINER" + - "P1_SCRIPT_REUSE_OVER_DUPLICATION" + - "P1_HARDCODED_TOOL_VERSION_IN_DOCKERFILE" + - "P1_DOCKER_LAYER_CACHING_DEPENDENCY_FIRST" + references: + - "PR #650" + - "PR #707" +``` + +--- + +## Review Checklist + +Before approving, verify: + +**Security & Credentials** +- [ ] **No debug/tunnel URLs** (P0_DEBUG_URL_IN_PRODUCTION_CONFIG): No lhr.life, ngrok, or localtunnel hostnames in any `.groovy` or config file +- [ ] **No credential printing** (P0_SENSITIVE_DEBUG_PRINT_IN_PIPELINE): No `cat`, `echo`, or `print` of SSH key or token file contents +- [ ] **No hardcoded IPs** (P0_HARDCODED_IP_IN_PIPELINE): All service endpoints use DNS names, not bare IP literals + +**Config & Template Safety** +- [ ] **Config injection safety** (P0_CONFIG_TEMPLATE_INJECTION_SAFETY): User-supplied data embedded in YAML templates is sanitized against multiline and special-character edge cases +- [ ] **No feature branches in env maps** (P1_FEATURE_BRANCH_IN_ENVIRONMENT_MAP): `environment_map` and `branch_param_map` contain only permanent branch names + +**Build Reliability** +- [ ] **No per-build tool downloads** (P1_EXTERNAL_BINARY_DOWNLOAD_IN_BUILD): sonar-scanner, Go tarball, or other tools are not curl-downloaded inside the stage body +- [ ] **Binary integrity** (P1_DOWNLOADED_BINARY_HASH_VERIFICATION): Any downloaded binary has a hash/checksum verification step, or comes from Artifactory +- [ ] **Prerequisite file checks** (P1_MISSING_PREREQUISITE_FILE_CHECK): Required config files (sonar-project.properties, etc.) are checked for existence before stages that depend on them +- [ ] **Test failures fail the build** (P1_TEST_FAILURE_MUST_FAIL_PIPELINE): Non-zero test exit codes propagate as pipeline failures; no silent suppression +- [ ] **Tools in base image** (P1_TOOLS_IN_BASE_IMAGE_NOT_BUILD_STEP): Global tools (sonar-scanner, pnpm, librdkafka) are in the base image, not installed per-build + +**Docker & Image Quality** +- [ ] **Dependency-first Dockerfile layering** (P1_DOCKER_LAYER_CACHING_DEPENDENCY_FIRST): Manifests (Cargo.toml, package.json) are copied and deps fetched before `COPY . .` +- [ ] **Parameterized tool versions** (P1_HARDCODED_TOOL_VERSION_IN_DOCKERFILE): Language/runtime version is a build ARG, not hardcoded in FROM +- [ ] **No runtime Dockerfile mutation** (P1_DYNAMIC_DOCKERFILE_MUTATION_VIA_SHELL): No `echo '...' >> Dockerfile-*` patterns in pipeline code +- [ ] **Correct env vars for runtime** (P1_WRONG_ENV_VAR_IN_CONTAINER): No Go-specific vars (GOMAXPROCS) in non-Go containers; template env vars audited for relevance + +**Code Architecture** +- [ ] **Centralized cleanup** (P1_CENTRALIZED_CLEANUP_IN_FINALLY): New failure paths route through the shared `finally` cleanup function, not ad-hoc inner try-catches +- [ ] **Precise flag semantics** (P1_SEMANTIC_FLAG_REUSE): New pipeline behaviors use new, clearly named flags; not piggybacking on `hot_fix` or `skip_sonar` +- [ ] **Shared constants** (P1_SHARED_CONSTANTS_FOR_REGISTRY_PATHS): Registry URLs and infra paths are defined as shared constants, not repeated inline +- [ ] **Script reuse** (P1_SCRIPT_REUSE_OVER_DUPLICATION): Config validation logic extends the existing canonical script; no new duplicate validation scripts + +--- + +## CodeRabbit-Sourced Learnings + +> Derived from CodeRabbit automated review comments across PRs #664, #727, and related. +> Merged into this file per AI Blitz Task 10. + +```yaml +- id: CR_UNDEFINED_VARIABLE_IN_NEW_STAGE + severity: P0 + title: New config fields must be defined in every build stage that references them + trigger_paths: + - "src/com/meesho/stages/buildRust.groovy" + - "src/com/meesho/stages/buildGo.groovy" + - "src/com/meesho/stages/buildMaven.groovy" + - "src/com/meesho/stages/buildNode.groovy" + - "src/com/meesho/stages/buildGradle.groovy" + required_evidence: + - "Every variable used in conditional logic (e.g. `repoType`, `skip_npmrc`) is extracted from `config` with a default value before first use" + - "Compare with existing build stages to ensure parity: if buildGo defines `def repoType = config.repoType ?: 'unknown'`, buildRust must too" + suggested_fix: + - "Add `def repoType = config.repoType ?: 'unknown'` near the other config extractions in the function" + why: > + CodeRabbit flagged in PR #664 that `repoType` was used in buildRust.groovy + without being defined, causing a MissingPropertyException at runtime. The variable + was properly defined in buildGo, buildGradle, and buildMaven but missed in the + new Rust stage. + references: + - "PR #664 (CodeRabbit)" + +- id: CR_INVERTED_CONDITIONAL_ON_REPOTYPE + severity: P0 + title: Docker build conditionals on repoType must use == not != for the intended type + trigger_paths: + - "src/com/meesho/stages/buildMaven.groovy" + - "src/com/meesho/stages/buildGo.groovy" + - "src/com/meesho/stages/buildNode.groovy" + - "src/com/meesho/stages/buildGradle.groovy" + - "src/com/meesho/stages/buildRust.groovy" + required_evidence: + - "Conditionals guarding Docker build (`stageDocker`) use `repoType == 'microservice'` (not `!=`)" + - "Both the GCP and fallback/AWS branches have the same guard polarity" + suggested_fix: + - "Replace `repoType != 'microservice'` with `repoType == 'microservice'` in the stageDocker conditional" + why: > + CodeRabbit caught in PR #664 that `repoType != 'microservice'` caused Docker + builds to run for library/client repos and skip for microservices — the exact + opposite of the PR's intent. The same inversion appeared in both the GCP and + fallback code paths. + references: + - "PR #664 (CodeRabbit)" + +- id: CR_GROOVY_SINGLE_QUOTE_NO_INTERPOLATION + severity: P1 + title: Log messages with variable references must use double quotes for GString interpolation + trigger_paths: + - "src/com/meesho/stages/*.groovy" + - "src/com/meesho/utilities/*.groovy" + required_evidence: + - "No log.info/log.warn/log.error calls use single-quoted strings containing `${...}` placeholders" + - "All strings with variable interpolation use double quotes (`\"...\"`)" + suggested_fix: + - "Change `log.info('Skipping for ${repoType}')` to `log.info(\"Skipping for ${repoType}\")`" + why: > + CodeRabbit flagged in PR #664 that single-quoted Groovy strings do NOT support + interpolation — `'${repoType}'` prints the literal text `${repoType}`, not its value. + Double-quoted strings become GStrings and evaluate `${}` expressions. + references: + - "PR #664 (CodeRabbit)" + +- id: CR_MISLEADING_SKIP_LOG_MESSAGE + severity: P1 + title: Skip/bypass log messages must state the actual reason for skipping + trigger_paths: + - "src/com/meesho/stages/buildGo.groovy" + - "src/com/meesho/stages/buildMaven.groovy" + - "src/com/meesho/stages/buildNode.groovy" + required_evidence: + - "Log messages in else/skip branches mention the actual condition that triggered the skip (e.g. `shouldValidateConfig=false` or `CHANGE_ID not set`)" + - "Messages do not blame `repoType` when the skip is caused by a different condition" + suggested_fix: + - "Include the actual condition values: `log.info(\"Skipping CAC validation (repoType=${repoType}, changeId=${env.CHANGE_ID ?: 'N/A'})\")`" + why: > + CodeRabbit flagged in PR #664 that the else branch logged "Skipping for repoType" + when the actual reason was that shouldValidateConfig returned false or CHANGE_ID + was not set. Misleading skip messages make debugging pipeline behavior harder. + references: + - "PR #664 (CodeRabbit)" + +- id: CR_DEFAULT_FLAG_BREAKS_EXISTING_BEHAVIOR + severity: P1 + title: New boolean config flags must default to preserving existing behavior + trigger_paths: + - "src/com/meesho/stages/buildNode.groovy" + - "src/com/meesho/stages/buildMaven.groovy" + - "src/com/meesho/stages/buildGo.groovy" + required_evidence: + - "New flags (e.g. `skip_npmrc`, `keep_package_lock`) default to the pre-existing behavior when not explicitly set" + - "If existing behavior is to include `.npmrc`, the default must be `skip_npmrc: false`, not `true`" + suggested_fix: + - "Change `def skip_npmrc = config.skip_npmrc != null ? config.skip_npmrc : true` to default `false` to preserve existing behavior" + why: > + CodeRabbit flagged in PR #727 that defaulting skip_npmrc to true would break + repos relying on private registries that need the secret-managed .npmrc. The + shared library serves all services — a new opt-in flag must not change default + behavior for existing consumers. + references: + - "PR #727 (CodeRabbit)" + +- id: CR_PACKAGE_LOCK_REMOVAL_VS_NPM_CI + severity: P1 + title: Removing package-lock.json is incompatible with npm ci as the default install command + trigger_paths: + - "src/com/meesho/stages/buildNode.groovy" + - "resources/com/meesho/node-Dockerfile" + required_evidence: + - "If `keep_package_lock=false` removes the lock file, `npm_install_arg` must not be empty (which defaults to `npm ci` in the Dockerfile)" + - "The Dockerfile fallback install command and the Groovy-side lock file removal logic are in sync" + suggested_fix: + - "When `keep_package_lock=false` and `npm_install_arg` is empty, set `npm_install_arg = 'npm install'` to avoid `npm ci` failing on missing lockfile" + why: > + CodeRabbit flagged in PR #727 that removing package-lock.json while leaving + npm_install_arg empty causes the Dockerfile to fall back to `npm ci`, which + requires a lockfile and will fail. The Groovy stage and Dockerfile must agree + on the install strategy. + references: + - "PR #727 (CodeRabbit)" +``` + +### CodeRabbit Review Checklist Additions + +Before approving, also verify: + +**Variable & Conditional Correctness (CodeRabbit-sourced)** +- [ ] **No undefined config variables** (CR_UNDEFINED_VARIABLE_IN_NEW_STAGE): Every variable used in a build stage is extracted from `config` with a default before first use +- [ ] **Correct conditional polarity** (CR_INVERTED_CONDITIONAL_ON_REPOTYPE): Docker build guards use `==` for the intended type, not `!=` +- [ ] **Double quotes for interpolation** (CR_GROOVY_SINGLE_QUOTE_NO_INTERPOLATION): No `log.info('${var}')` — must be `log.info("${var}")` +- [ ] **Accurate skip messages** (CR_MISLEADING_SKIP_LOG_MESSAGE): Skip/bypass logs state the actual condition, not a misleading one +- [ ] **Safe flag defaults** (CR_DEFAULT_FLAG_BREAKS_EXISTING_BEHAVIOR): New boolean config flags default to preserving pre-existing behavior +- [ ] **Lock file vs install command sync** (CR_PACKAGE_LOCK_REMOVAL_VS_NPM_CI): Removing lock files is paired with a compatible install command diff --git a/review.md b/review.md new file mode 100644 index 0000000..3769d2c --- /dev/null +++ b/review.md @@ -0,0 +1,61 @@ +<!-- This file captures service-specific code review knowledge that only your team knows. + Write concrete, actionable rules under each heading — the AI reviewer will use these + as mandatory context for every PR in this repo. + Keep this file under 200 lines — be specific, not exhaustive. --> + +## How to Review in This Service + +<!-- List the data rules that must ALWAYS hold in this service — things that would break + the business or corrupt data if violated. + For each invariant, state the entity, the constraint, and what to flag in a diff. --> +**Domain invariants** +- Whitelist gates in `constructParam.groovy:getWhitelistedRepos` clone `Meesho/whitelists` fresh on every call (no caching). This is deliberate — each clone captures latest state so a DevOps whitelist change takes effect on the next build without a library release. Reject any caching/memoization that doesn't replace the freshness story. +- The literal `"ringmaster-bot"` userId in `deployRingmaster.groovy:run` is the only signal that distinguishes Ringmaster callbacks from Turbo-Turtle callbacks. Reject any rename of the string without explicit Ringmaster-team coordination. +- `buildObjHelper.groovy:run` falls through to `defaultBuild` silently when no `toolchain` case matches — a mis-spelled toolchain (`golang` vs `go`) produces a no-op build that "succeeds" with no artifact. Flag any new builder added without a match arm, and any toolchain string the dispatcher doesn't recognise. +- JVM `-Xmx`/`-Xms` are auto-calculated in `deployArgoCD.groovy:update_helm_repo` from `memory_request` (`xmx = memory_request * 0.75`, `xms = xmx * 0.5`). Reject hardcoded `-Xmx` in `JAVA_OPTS` unless `jvm_memory_override: true` is also set in the service's `deployment.yaml`. +- Services with `priority_v2: sp0` or `up0` MUST have canary configured — `deployArgoCD.groovy:run` blocks non-canary `prd` deploys for these priorities and there is no bypass. Reject any diff that adds one. +- `environment_map` / `branch_param_map` in `constructParam.groovy` and `deployArgoCD.groovy` may only contain permanent branches (`master`, `main`, `develop`, `gcp-main`). Flag any `feature/...` entry — these propagate to every consumer (P1_FEATURE_BRANCH_IN_ENVIRONMENT_MAP, PR #707). +- `constructTemplate.groovy:_construct()` is `@NonCPS` because it uses Java regex and string interpolation that aren't CPS-serialisable. Reject removal of the annotation or moves of its logic into a CPS context (inlining into `stage` bodies or `parallel` closures). +- The four-step ArgoCD deploy order in `deployArgoCD.groovy` (`update_argo_repo → refresh_app_of_apps → update_helm_repo → refresh_and_sync`) is load-bearing — first-deploy services rely on `refresh_app_of_apps` creating the `Application` object. Steps 2 and 4 are not interchangeable. + +<!-- List the high-value request paths a reviewer should trace end-to-end. + For each workflow, state the entry point (route/controller), the handler chain, and + why it matters (latency-sensitive, money path, etc.). --> +**Critical workflows** +- `vars/buildPipeline.groovy:call` is the primary entry point invoked by every consuming `Jenkinsfile`. The chain is `constructParam.run → checkOut.run → buildObjHelper.run(build_tool) → notify.run`, wrapped in `try/catch/finally`. The `finally` block is the only thing that guarantees Slack notification on failure — trace any change to ordering, exception handling, or the surrounding `try/catch/finally`. +- `vars/gkeCICD.groovy`, `vars/eksCICD.groovy`, `vars/cloudFunctionCICD.groovy`, `vars/gcpMigration.groovy`, `vars/onlyPushtoJfrog.groovy` are sibling entry points — verify any change to a shared helper (`constructParam`, `notify`, `deployArgoCD`, `buildObjHelper`) is consistent across all of them. +- `deployArgoCD.groovy:run` is the prd-deploy critical path: priority check → 4-step deploy. A break here breaks every service's prd rollout — trace canary enforcement and the deploy-step ordering. +- `deployRingmaster.groovy:run` is the prd callback path: every prd deploy depends on the curl-to-Turbo-Turtle / Ringmaster handoff. Payload is written to a temp file and sent via `curl -d @<file>` to avoid bash escaping — never inline new fields into the curl command. +- `constructParam.groovy:getWhitelistedRepos` is consulted for five gates (`skip-sonar`, `app-config-disabled`, `multizone-enabled`, `allowedNonDevelopPrDeployment`, `ValidateCacConfig`). Any change to gate semantics must be traced end-to-end through all five callers. + +<!-- List bugs, outage patterns, or recurring mistakes this service has hit before. + For each pattern, describe the failure, root cause, and what to watch for in diffs. --> +**Known failure patterns** +- Debug tunnel hostnames (`*.lhr.life`, `*.ngrok.io`, `*.tunnel.*`) merged into `deployRingmaster.groovy` after local testing (PRs #596, #664) — flag any tunnel literal in any `.groovy` or YAML (P0_DEBUG_URL_IN_PRODUCTION_CONFIG). +- SSH private-key contents printed to Jenkins console via `cat ./id_github_jenkins` in `addSSHKey.groovy` (PR #634). Jenkins console is readable by anyone with job-read access — flag any `cat`/`echo`/`print`/`ls -al` of credential paths (P0_SENSITIVE_DEBUG_PRINT_IN_PIPELINE). +- Bare IPs (`172.x.x.x`, `10.x.x.x`) shipped as build-callback endpoints in `notify.groovy` (PR #681). IPs break silently when infra is rebalanced — flag any IP literal, replace with DNS (P0_HARDCODED_IP_IN_PIPELINE). +- Dockerfiles mutated at build time via `sh "echo '...' >> Dockerfile-${artifactId}"` (PR #707). Runtime mutation makes the actual image instructions invisible in code review — use `ARG`/multi-stage or a dedicated template (P1_DYNAMIC_DOCKERFILE_MUTATION_VIA_SHELL). +- Test failures swallowed via `if (testExitCode != 0) { echo 'pipeline continues' }` patterns (PR #643). Broken code packaged and deployed while CI appears to pass — flag any test-exit-code suppression (P1_TEST_FAILURE_MUST_FAIL_PIPELINE). +- Shipped typos and copy-paste bugs that Groovy/Jenkins doesn't catch at compile time: `chekoutSubmodule` in `checkOut.groovy:22`, `catch (Exceptione)` in `helmGenerator.groovy:102`, duplicate `buildRegistry` map key in `buildPython.groovy:76-79`, hard-coded `branch_name = 'repo'` in `buildGradle.groovy:252`. Reviewer is the only gate — read identifier names and map keys carefully. +- New language support created by copy-pasting an existing language's template without auditing every field (PR #650 left `GOMAXPROCS` in `rust-values.yaml`; PR #707 duplicated `validate_configs_v2.py` into `validate_configs_node.py`). Flag any new template file that mirrors an existing one — every env var, ARG, and shared script must be audited. + +<!-- List hot paths, SLA-sensitive operations, resource constraints, and performance + conventions specific to this service. Mention concrete thresholds where they exist. --> +**Performance considerations** +- `getWhitelistedRepos` does up to 5 fresh `git clone` of `Meesho/whitelists` per build under GitHub rate-limiting. Do not add a sixth gate without coordinating with infra; do not add caching without a replacement freshness story (see Domain invariants). +- Per-build downloads of `sonar-scanner`, Go toolchain, `pnpm`, `librdkafka` add external-network dependency to the critical path and re-fetch on every build (PRs #306, #643). Bake into the builder base image (P1_TOOLS_IN_BASE_IMAGE_NOT_BUILD_STEP, P1_EXTERNAL_BINARY_DOWNLOAD_IN_BUILD). +- Dockerfile layer caching: dependency manifests (`Cargo.toml`/`Cargo.lock`, `package.json`/`package-lock.yaml`, `go.mod`/`go.sum`, `pom.xml`) must be `COPY`-ed and dependencies fetched before `COPY . .` in every Dockerfile under `resources/com/meesho/` (P1_DOCKER_LAYER_CACHING_DEPENDENCY_FIRST, PR #650). +- Non-prd node pools are BU-shared (`nodePoolSelection.groovy:run` → `{BU}-shared`). A memory leak or CPU spike on one staging/dev service degrades every other service in the same BU — flag any material increase in `requests`/`limits` on a non-prd values.yaml without BU-owner sign-off. +- Any externally downloaded binary must have hash verification (`sha256sum -c <expected>`) or be replaced by an Artifactory-hosted artifact with controlled provenance (P1_DOWNLOADED_BINARY_HASH_VERIFICATION, PR #643). +- Registry base URLs (`asia-southeast1-docker.pkg.dev/meesho-central-dev-0622/toolchain`) must be referenced via a shared constant in `src/com/meesho/utilities/`, not repeated inline across stage files (P1_SHARED_CONSTANTS_FOR_REGISTRY_PATHS, PR #681). + +<!-- List external systems this service talks to and the contracts/quirks a reviewer + should know. Include auth, messaging, config backends, and their failure modes. --> +**Integration boundaries** +- ArgoCD (`deployArgoCD.groovy`): 4-step deploy order is load-bearing; `argocd login --password ${ARGO_PASSWORD}` leaks the password via process list — prefer stdin or env-var injection. Flag any new `--password <literal>` style invocation. +- Turbo-Turtle / Ringmaster callbacks (`deployRingmaster.groovy`): canonical internal DNS endpoint (e.g. `turbo-turtle.meeshogcp.in`); payload sent via `curl -d @<tempfile>`; discriminated by `getCause(UserIdCause).getUserId() == "ringmaster-bot"`; temp file deleted in a `finally` block. +- GitHub SCM: `env.CHANGE_ID` from the GitHub Branch Source plugin is the canonical PR-build detector; never use `env.BRANCH_NAME =~ /PR-/` — it breaks on non-GitHub SCMs and on re-triggered builds. `Meesho/whitelists` is cloned freshly per gate. +- JFrog / S3 artifact targets: `push_to_jfrog` / `push_to_s3` are the boolean params that allow non-master branches to push — verify any new pusher honours both flags. +- Sonar: `skip_sonar` whitelist via `skip-sonar-whitelist.yaml`; uses the per-call `Meesho/whitelists` clone. Never overload `skip_sonar` (or `hot_fix`) for unrelated semantics — introduce a new, clearly named flag instead (P1_SEMANTIC_FLAG_REUSE, PRs #495, #578). +- Slack `#ci-cd-status` (default; override via `notify_channel`): notification fired from `notify.run` inside the top-level `finally` block of `buildPipeline.groovy`. Verify the `maintainer` Slack username is honoured on failure paths. +- GCP Artifact Registry `asia-southeast1-docker.pkg.dev/meesho-central-dev-0622/toolchain`: canonical toolchain registry. A project or region migration requires updating one constant — flag any inline duplicate of the URL. diff --git a/sandbox-settings.xml b/sandbox-settings.xml new file mode 100644 index 0000000..e69de29 diff --git a/schemas/asyncapi.yml b/schemas/asyncapi.yml new file mode 100644 index 0000000..e69de29 diff --git a/src/com/meesho/stages/automationTest.groovy b/src/com/meesho/stages/automationTest.groovy new file mode 100644 index 0000000..b261682 --- /dev/null +++ b/src/com/meesho/stages/automationTest.groovy @@ -0,0 +1,68 @@ +package com.meesho.stages + +import com.meesho.stages.deployJar + + +def run(Map config){ + // get required variables from config + def automation_repo_name = config.run_automation.repo_name + def branch = config.run_automation.branch + def deployObj = new deployJar() + + //deploy service(farmiso) on specified machine + deployObj.run(config) + + + checkoutAutomationRepo(automation_repo_name, branch) + + runAutomationSuite(automation_repo_name) + +} + +def checkoutAutomationRepo(String automation_repo_name, String branch){ + try{ + stage('Checkout automation repo'){ + sh "rm -rf ${automation_repo_name}; git clone git@github.com:Meesho/${automation_repo_name}.git -b ${branch}" + echo "automation repo cloned" + } + } + catch( Exception e) { + env.msg = "Error cloning automation repo. Please check console output for more details." + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +def runAutomationSuite(String automation_repo_name){ + try{ + stage('Run Automation Suite'){ + dir("$automation_repo_name"){ + run_cmd = "mvn test" //specific to Farmiso as of now. + sh "$run_cmd" + } + } + } + catch( Exception e) { + env.msg = "Error running automation suite. Please check console output for more details." + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + finally{ + //publish HTML test report + dir("$automation_repo_name"){ + archiveArtifacts artifacts: "test-output/*.*" + + + publishHTML (target: [ + allowMissing: false, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'test-output', + reportFiles: 'index.html', + reportName: "Farmiso-Test-Automation-Report" + ]) + } + } +} diff --git a/src/com/meesho/stages/buildDocker.groovy b/src/com/meesho/stages/buildDocker.groovy new file mode 100644 index 0000000..d068865 --- /dev/null +++ b/src/com/meesho/stages/buildDocker.groovy @@ -0,0 +1,68 @@ +package com.meesho.stages + +import com.meesho.stages.deployArgoCD +import com.meesho.stages.buildObjHelper +import com.meesho.utilities.buTeamMapping + +def run(Map config) { + def btObj = new buTeamMapping() + def helperObj = new buildObjHelper() + def buildObj = helperObj.run(config.dockerBuildVersion) + def (tag , deployArgo) = buildObj.buildDckr(config) + release(config, tag, deployArgo) +} + +def release(Map config, String tag, boolean deployArgo) { + //TODO: move params to run function and move argoCD call there as well. + def btObj = new buTeamMapping() + def repo_name = config.repo_name + def branch_name = "${env.BRANCH_NAME}" + def team = btObj.get_team_initials(config.team) + env.TAG = tag + + log.info("tag is ${tag}") + + def bu = btObj.get_bu_initials(config.bu) + def notify_channel = config.notify_channel && config.notify_channel != '' ? config.notify_channel : 'canary-status' + validate('bu', bu, config.bu) + validate('team', team, config.team) + def build_user = currentBuild.rawBuild.getCause(Cause.UserIdCause).getUserId() + env.deployArgo = deployArgo + // Calling Ringmaster + if (build_user != "ringmaster-bot" && build_user!="turbo-turtle" && build_user != "toolchain-jenkins"){ + if (env.cicd_environment == 'ftr') { + log.warning("**********!!!!!! Deployments in '${env.cicd_environment}' env are not supported !!!!!!**********") + log.info("****** To deploy in Staging, Please merge the changes to your 'develop'/'development' branch and deploy ******") + return + } + if (deployArgo) { + def argoCd_obj = new deployArgoCD() + def deployment_order = config.deployment_order + def dockerBuildVersion = config.dockerBuildVersion + argoCd_obj.run(repo_name, deployment_order, tag, team, dockerBuildVersion, notify_channel) + } else { + log.info("Skipping ArgoCD deployment for ${repo_name}") + } + + if (config.distribution_id) { + stage(stageName('Refreshing CDN file')) { + def folder_name = config.chunk_base_path ?: repo_name.split('_')[-1] + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws cloudfront create-invalidation --distribution-id ${config.distribution_id} --paths \"/${folder_name}/remoteEntry.js\"") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + // echo "Skipping - CDN Invalidation" + sh(script:"gcloud config set project ${env.GCPLBProject};gcloud compute url-maps invalidate-cdn-cache '${config.distribution_id}' --path \"/${folder_name}/remoteEntry.js\" --project='${env.GCPLBProject}' --async") + } + } + } + } +} + +def validate(String name, String value, String printValue) { + if (value == null) { + log.error("Incorrect ${name} name ${printValue}") + currentBuild.result = env.FAILURE + return + } +} diff --git a/src/com/meesho/stages/buildGo.groovy b/src/com/meesho/stages/buildGo.groovy new file mode 100644 index 0000000..c1f0ec6 --- /dev/null +++ b/src/com/meesho/stages/buildGo.groovy @@ -0,0 +1,412 @@ +package com.meesho.stages + +import com.meesho.utilities.buTeamMapping +import com.meesho.utilities.constructTemplate +import com.meesho.utilities.getDockerParams +import com.meesho.utilities.addSSHKey +import com.meesho.utilities.gitActions +import com.meesho.utilities.constructParam +import com.meesho.utilities.dockerUtilities + + +def buildDckr(Map config) { + env.GOPRIVATE = 'github.com/Meesho' + def buTeamMappingObj = new buTeamMapping() + def constructObj = new constructTemplate() + def dockerParamObj = new getDockerParams() + def addSshKeyObj = new addSSHKey() + def dockerUtilObj = new dockerUtilities() + + def team = buTeamMappingObj.get_team_initials(config.team) + def modules = config.modules ?: ['module_less'] + def repoName = config.repo_name + def dockerRepository = "${env.cicd_environment}/${team}/${repoName.toLowerCase()}" + def tag = dockerParamObj.getTag(repoName) + def dockerBindings = [:] + def version = config.dockerBuildVersion.split('-')[-1] + def constructParamObj = new constructParam() + def repoType = config.repo_type ?: 'microservice' + boolean skipSonarAndQualityGate = constructParamObj.skipSonarCheckForGo(config) + echo "skipSonarAndQualityGate: ${skipSonarAndQualityGate}" + String goVersion = config.goVersion ?: version + boolean shouldDeployArgo = (config.deployArgo ?: false).toBoolean() + boolean isConfigOnlyChange = false + + // Ensure goVersion is in x.x.x format + echo "goVersion: ${goVersion}" + if (goVersion.split('\\.').size() == 2) { + goVersion += '.0' + } + if (env.INFRA_ENV == 'toolchain') { + tag = dockerParamObj.getTag(repoName) + boolean allImagesExist = true + + for (module in modules) { + def moduleName = (module instanceof LinkedHashMap) ? module.keySet()[0] : module + def modulePath = (moduleName == 'module_less') ? dockerRepository : "${dockerRepository}/${moduleName}" + //TODO: building everything if one is missing -> create list of modules unbuild will only build those + if (!dockerUtilObj.imageExists(env.registry, modulePath, tag)) { + log.info("Toolchain: Image missing for ${moduleName} at ${modulePath}:${tag}. Proceeding with build.") + allImagesExist = false + break + } + } + if (allImagesExist) { + log.info("Toolchain: All images found in registry for tag ${tag}. Skipping build step.") + return [tag, shouldDeployArgo] + } + } + stage(stageName('Check Config Changes and Get Latest Image Tag')) { + (isConfigOnlyChange, shouldDeployArgo) = is_config_only_change_and_should_deploy_argo(repoName, shouldDeployArgo) + log.info("Is just application config change? $isConfigOnlyChange") + def param = new constructParam() + if (env.CHANGE_ID && repoType == 'microservice') { + def shouldValidateConfig = param.ValidateCacConfigForRepo(false, repoName) + if (shouldValidateConfig) { + log.info('************ Validate Config for CAC application.yml files ************') + dir("$repoName") { + writeFile file: 'validate_configs.py', text: libraryResource('com/meesho/validate_configs_v2.py') + sh 'python3 validate_configs.py' + } + } else { + log.info('************ Skipping Validation of CAC Config ************') + } + } else { + log.info("Skipping Validation of CAC Config for ${repoType} repo type") + log.info('************ Skipping Validation of CAC Config ************') + } + + if (isConfigOnlyChange) { + log.info('Skipping the build, since it is just application config change') + log.info('Getting latest image tag from GAR') + def firstModule = modules[0] + def firstModuleName = (firstModule instanceof LinkedHashMap) ? firstModule.keySet()[0] : firstModule + def moduleDockerRepository = (firstModuleName == 'module_less') ? dockerRepository : "${dockerRepository}/${firstModuleName}" + try { + def latest_image = sh( + script: "gcloud container images list-tags ${env.registry}/${moduleDockerRepository} --format='value(tags)' | sed '/^\$/d' | awk -F'-' '{print \$NF}' | sort | tail -1", + returnStdout: true + ).trim() + echo "latest_image: ${latest_image}" + if (latest_image == '') { + log.info("No existing images found for ${env.registry}/${moduleDockerRepository}. Using generated tag: ${tag}") + } else { + tag = sh( + script: "gcloud container images list-tags ${env.registry}/${moduleDockerRepository} --format='value(tags)' | tr ',' '\n' | grep ${latest_image}", + returnStdout: true + ).trim() + } + } catch (Exception e) { + env.msg = 'Error getting latest docker image from GAR' + env.error_msg_to_db = env.msg + log.error("${env.msg}\n${e.toString()}") + currentBuild.result = env.FAILURE + throw e + } + return [tag, shouldDeployArgo] + } + log.info('Proceeding with the build') + } + + stage(stageName('Scanning Sonar and Quality Gate')) { + dir(repoName) { + sonar_scan(repoName, skipSonarAndQualityGate, goVersion) + } + } + if (currentBuild.result == 'UNSTABLE') + { return [tag, false] } + + if (repoType != 'microservice') { + log.info("Skipping Docker build for ${repoType} repo type") + return [tag, shouldDeployArgo] + } + + stage(stageName('Building docker images')) { + try { + sh(script: 'gcloud auth configure-docker asia-southeast1-docker.pkg.dev --quiet') + } catch (Exception e) { + env.msg = "[Failure] Can't login to gcloud docker registry." + env.error_msg_to_db = env.msg + log.error("${env.msg}. Error: ${e.toString()}") + currentBuild.result = env.FAILURE + throw e + } + dockerBindings['version'] = (version == 'go') ? '1.24.4' : version + dockerBindings['base_dir'] = config.base_dir ?: false + dockerBindings['build_registry'] = env.buildRegistry + dockerBindings['go_proxy'] = env.goProxyUrl + dockerBindings['repo_name'] = repoName + if (config.containsKey('copy_file')) { + String recursive = config.copy_file.recursive ? ' -r' : '' + dir(repoName) { + dir('copied_files') { + sh(script: "gsutil cp${recursive} ${config.copy_file.path} .") + } + } + dockerBindings['copy_file'] = true + dockerBindings['copy_target'] = config.copy_file.target ?: '/app/' + dockerBindings['base_dir'] = config.base_dir ?: false + } else { + dockerBindings['copy_file'] = false + } + + try { + dir(repoName) { + addSshKeyObj.create() + if (!fileExists('Dockerfile')) { + def moduleBuilds = [:] + for (m in modules) { + def moduleRef = m + def moduleName = (moduleRef instanceof LinkedHashMap) ? moduleRef.keySet()[0] : moduleRef + moduleBuilds["build-${moduleName}"] = { + def localBindings = dockerBindings.clone() + localBindings['module_property'] = (moduleRef instanceof LinkedHashMap) ? moduleRef[moduleName] : [:] + localBindings['kafka'] = (localBindings['module_property'].kafka) ? '-kafka' : '' + localBindings['module'] = moduleName + constructObj.renderTemplate(localBindings, 'go-Dockerfile', "Dockerfile-${moduleName}") + sh "cat Dockerfile-${moduleName}" + def moduleDockerRepository = (moduleRef == 'module_less') ? dockerRepository : "${dockerRepository}/${moduleName}" + sh(script: 'tar -cf only-mods.tar $(git ls-files "go.mod" "go.sum" "**/go.mod" "**/go.sum" 2>/dev/null || find . -name go.mod -o -name go.sum)') + sh(script: "set +x && docker build --tag ${env.registry}/${moduleDockerRepository}:${tag} -f Dockerfile-${moduleName} .") + if (env.cicd_environment != 'ftr' || env.INFRA_ENV == 'toolchain') { + dockerUtilObj.retryDockerPush("docker push ${env.registry}/${moduleDockerRepository}:${tag}") + } else { + log.info("Skipping Docker Push - ${env.cicd_environment} env") + } + sh(script: "rm -rf Dockerfile-${moduleName}") + } + } + parallel moduleBuilds + } else { + sh 'cat Dockerfile' + def moduleDockerRepository = dockerRepository // default for repo root + if (modules && modules[0] != 'module_less') { + def firstModule = modules[0] + def firstModuleName = (firstModule instanceof LinkedHashMap) ? firstModule.keySet()[0] : firstModule + moduleDockerRepository = dockerRepository + '/' + firstModuleName + } + def pushOrNot = (env.cicd_environment != 'ftr' || env.INFRA_ENV == 'toolchain') ? '--push' : '' + if (!pushOrNot) { + log.info("Skipping Docker Push - ${env.cicd_environment} env") + } + sh(script: "docker build --tag ${env.registry}/${moduleDockerRepository}:${tag} ${pushOrNot} .") + } + } + } catch (Exception e) { + env.msg = 'Error in building DockerFile Or Pushing To ECR' + env.error_msg_to_db = env.msg + log.error("${env.msg}. Error: ${e.toString()}") + currentBuild.result = env.FAILURE + throw e + } + return [tag, shouldDeployArgo] + } +} + +def is_config_only_change_and_should_deploy_argo(String repoName, boolean deployArgo) { + if (env.INFRA_ENV == 'toolchain') { + return [false, deployArgo] + } + def gitObj = new gitActions() + def configFiles = [] + boolean shouldDeployArgoResult = deployArgo + boolean isConfigOnlyChange = false + + def changedFiles = env.CHANGE_ID ? + gitObj.fetchDiffFilesForPullRequest(repoName, env.CHANGE_TARGET) : + gitObj.fetchDiffFilesForPushRequest(repoName) + + if (!changedFiles) { + return [isConfigOnlyChange, shouldDeployArgoResult] + } + + for (file in changedFiles.split('\n')) { + if (!file.startsWith('configs/')) { + isConfigOnlyChange = false + return [isConfigOnlyChange, shouldDeployArgoResult] + } + isConfigOnlyChange = true + configFiles << file + } + + if (configFiles && deployArgo && isConfigOnlyChange) { + def hasEnvironmentConfig = configFiles.any { it.contains(env.cicd_environment) } + if (!hasEnvironmentConfig) { + shouldDeployArgoResult = false + echo "Config changes don't contain environment: ${env.cicd_environment}" + } + } + return [isConfigOnlyChange, shouldDeployArgoResult] +} + +/** + * Function to execute sonar scan + * @param repoName: repository name + * @param skipSonarAndQualityGate: boolean parameter to skip sonar scan + * @param goVersion: Go version string + */ +def sonar_scan(String repoName, boolean skipSonarAndQualityGate, String goVersion) { + if (skipSonarAndQualityGate) { + log.info('Sonar scan is skipped. Marking this stage as passed.') + return + } + stage(stageName('Running sonar scan')) { + try { + withSonarQubeEnv(env.sonarEnv) { + echo "env.sonarEnv: ${env.sonarEnv}" + echo "env.BRANCH_NAME: ${env.BRANCH_NAME}" + def exclusions = "" + if (!fileExists('sonar-project.properties')) { + exclusions = " -Dsonar.exclusions=**/*_test.go,**/mock_*.go,**/mock.go,**/*.pb.go,**/*.proto,**/model.go" + }else{ + exclusions = " -Dproject.settings=`pwd`/sonar-project.properties" + } + + def scannerCommand = "sonar-scanner -Dsonar.projectKey=${repoName} -Dsonar.go.coverage.reportPaths=./cov.out -Dsonar.branch.name=${env.BRANCH_NAME} -Dsonar.ws.timeout=120 ${exclusions}" + echo "env.SONAR_HOST_URL: ${env.SONAR_HOST_URL}" + env.PATH = "/usr/local/sonar-scanner/sonar-scanner-5.0.1.3006-linux/bin:${env.PATH}" + def response = sh( + script: "curl -s -w '\n%{http_code}' -u ${env.SONAR_AUTH_TOKEN}: ${env.SONAR_HOST_URL}/api/navigation/component?component=${repoName}", + returnStdout: true + ).trim() + + def responseLines = response.split('\n') + def statusCode = responseLines[-1] + def responseBody = responseLines[0..-2].join('\n') + + log.info("SonarQube API Response Code: ${statusCode}") + log.info("SonarQube API Response Body: ${responseBody}") + + def projectExists = (statusCode == '200') + if (!projectExists && env.CHANGE_ID) { + log.info("Project ${repoName} does not exist. Creating it via a bootstrap scan on the target branch.") + + def targetBranch = env.CHANGE_TARGET ?: 'develop' + echo "targetBranch: ${targetBranch}" + sh "sonar-scanner \ + -Dsonar.projectKey=${repoName} \ + -Dsonar.projectName=${repoName} \ + -Dsonar.branch.name=${targetBranch} \ + -Dsonar.sources=. \ + -Dsonar.scm.disabled=true \ + -Dsonar.ws.timeout=120" + log.info("Bootstrap complete. Project created.") + }else{ + echo "Project ${repoName} already exists. Skipping bootstrap scan." + } + + if (env.CHANGE_ID) { + echo "env.CHANGE_ID: ${env.CHANGE_ID}" + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh "git fetch origin ${env.CHANGE_TARGET}:refs/remotes/origin/${env.CHANGE_TARGET}" + } + scannerCommand = "sonar-scanner -Dsonar.projectKey=${repoName} -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.repository=Meesho/${repoName} -Dsonar.pullrequest.key=${env.CHANGE_ID} -Dsonar.pullrequest.branch=${env.CHANGE_BRANCH} -Dsonar.pullrequest.base=${env.CHANGE_TARGET} -Dsonar.go.coverage.reportPaths=./cov.out -Dsonar.ws.timeout=120${exclusions}" + echo "scannerCommand: ${scannerCommand}" + } + downloadGoFromJFrog(goVersion) + sh(script: "GOPROXY=${env.goProxyUrl},direct && go mod tidy") + int testExitCode = sh(script: 'go test -short -coverprofile=./cov.out ./...', returnStatus: true) + if (testExitCode != 0) { + error('Go tests failed.') + } + echo "scannerCommand: ${scannerCommand}" + sh(script: scannerCommand) + } + } catch (Exception e) { + log.error("Error in running sonar scan: ${e}") + currentBuild.result = env.FAILURE + throw e + } + } + stage(stageName('Quality Gate')) { + try { + if (!env.CHANGE_ID || env.cicd_environment == 'int') { + log.info('Skipping quality gate check on Branches/Pre-Prod. Marking this stage as passed.') + } else { + timeout(time: 600, unit: 'SECONDS') { + def qg = waitForQualityGate() + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + if (qg.status != 'OK') { + log.warning("Quality gate failed: ${qg.status}") + error "Stopping pipeline due to quality gate failure." + } + } + } + } + }catch (Exception e) { + echo "Error in quality gate: ${e.message}" + unstable('Quality Gate Failed !') + throw e + } + } + } + + +/** + * Function to download Go binary from JFrog + * @param goVersion: Go version string + */ +def downloadGoFromJFrog(String goVersion) { + try { + echo "env.cicd_environment: ${env.cicd_environment}" + + def jfrogUrl = (env.cicd_environment == 'prd' || env.cicd_environment == 'int') ? + 'https://jfrog-prd.meeshogcp.in' : + 'https://jfrog-dev.meeshogcp.in' + + def repo = 'devops-tools-local' + def goTarball = "go${goVersion}.linux-amd64.tar.gz" + def jfrogPath = "${jfrogUrl}/artifactory/${repo}/go/${goTarball}" + + def jfrogUser = '' + def jfrogPass = '' + def credentialId = (env.cicd_environment == 'prd' || env.cicd_environment == 'int') ? + 'jfrog-prd-credentials' : 'jfrog-stg-credentials' + echo "Using credential ID: ${credentialId}" + withCredentials([usernamePassword(credentialsId: credentialId, + usernameVariable: 'JFROG_USER', + passwordVariable: 'JFROG_PASS')]) { + jfrogUser = env.JFROG_USER + jfrogPass = env.JFROG_PASS + + log.info("Attempting to download Go ${goVersion} from JFrog: ${jfrogPath}") + + def downloadStatus = sh(script: """ + curl -u "${env.JFROG_USER}:${env.JFROG_PASS}" \ + -fLO "${jfrogPath}" \ + --fail --silent --show-error + """, returnStatus: true) + + if (downloadStatus == 0 && fileExists(goTarball)) { + log.info("Successfully downloaded Go ${goVersion} from JFrog") + } else { + echo("WARN: Go ${goVersion} not found in JFrog. Falling back to go.dev...") + sh(script: "curl -LO https://go.dev/dl/${goTarball}") + + log.info("Uploading downloaded Go ${goVersion} to JFrog for future use") + sh(script: """ + curl -u "${env.JFROG_USER}:${env.JFROG_PASS}" \ + -T "${goTarball}" \ + "${jfrogPath}" \ + --fail --silent --show-error || echo "Upload to JFrog failed, but continuing..." + """) + } + } + + sh(script: "tar -xvzf ${goTarball} -C /usr/local", returnStdout: true) + sh(script: "rm -Rf ${goTarball}") + env.PATH = "/usr/local/go/bin:${env.PATH}" + + // Verify installation + sh(script: "go version") + + } catch (Exception e) { + echo("ERROR: Failed to download Go from JFrog: ${e.message}") + + // Final fallback + sh(script: "curl -LO https://go.dev/dl/go${goVersion}.linux-amd64.tar.gz") + sh(script: "tar -xvzf go${goVersion}.linux-amd64.tar.gz -C /usr/local", returnStdout: true) + sh(script: "rm -Rf go${goVersion}.linux-amd64.tar.gz") + env.PATH = "/usr/local/go/bin:${env.PATH}" + } +} diff --git a/src/com/meesho/stages/buildGradle.groovy b/src/com/meesho/stages/buildGradle.groovy new file mode 100644 index 0000000..0eb2963 --- /dev/null +++ b/src/com/meesho/stages/buildGradle.groovy @@ -0,0 +1,542 @@ +package com.meesho.stages + +import com.meesho.utilities.buTeamMapping +import com.meesho.utilities.constructTemplate +import com.meesho.utilities.getDockerParams +import com.meesho.stages.checkOut + +/* +Function to define the flow of entire build, this function will call different stages related to maven build +*/ +def run(Map config) { + // get required variables from config + def checkoutObj = new checkOut() + def repo_name = config.repo_name + def args = config.build_args ?: '' + def skip_test = config.skip_test ?: false + def skip_sonar = config.skip_sonar ?: false + def push_to_jfrog = config.push_to_jfrog ?: false + def push_to_s3 = config.push_to_s3 ?: false + def branch_name = "${env.BRANCH_NAME}" + if (branch_name == 'gcp-main' || branch_name == 'gcp-master') { + push_to_jfrog = config.containsKey('push_to_jfrog') ? config.push_to_jfrog : false + } + def version = getVersion("${repo_name}") + + if (env.hot_fix) { + skip_test = true + skip_sonar = true + push_to_jfrog = false + } + + // call the stages + checkS3(repo_name, branch_name, version, push_to_s3) + build(repo_name, skip_test, args) + sonar_scan(repo_name, skip_sonar) + if (!env.CHANGE_ID) { + pushArtifactToJFrog(repo_name, push_to_jfrog) + pushArtifactToS3(repo_name, branch_name, push_to_s3) + } + else { + log.info('Artifact Push is disabled for Pull Requests') + } +} + +/* +Function to check if the artifact exist in s3 or not +input argument +*/ +def checkS3(String repo_name, String branch_name, String version, boolean push_to_s3) { + stage(stageName('Checking if artifact already exists')) { + if (env.CLOUD_PROVIDER == 'AWS') { + if (env.hot_fix) { + env.TAG = "v${version}-HOT" + } + else { + if (("${branch_name}" == 'master' || "${branch_name}" == 'main' || push_to_s3) && !env.CHANGE_ID) { + env.TAG = "v${version}" + log.info('########################## Checking if Artifact Already Exists. ###########################') + artifact_exists = sh(returnStdout: true, script: "aws s3 ls \"s3://${env.objBucket}/${repo_name}/${branch_name}/${TAG}/\" 2>/dev/null || echo ''").trim() + log.info("${artifact_exists}") + if ( artifact_exists ) { + env.msg = "CI for this\n version - ${version}\n branch - ${branch_name}\n TAG - ${TAG}\nis already done. Please proceed with CD." + log.error(env.msg) + currentBuild.result = 'FAILURE' + throw new Exception(env.msg) + } + else { + log.info("Proceeding with building artifact for TAG - ${TAG}.") + } + } + else { + log.info('Skipping - Building artifact') + } + } + } + else if (env.CLOUD_PROVIDER == 'GCP') { + if (env.hot_fix) { + env.TAG = "v${version}-HOT" + } + else { + if (("${branch_name}" == 'master' || "${branch_name}" == 'main' || "${branch_name}" == 'gcp-main' || "${branch_name}" == 'gcp-master' || push_to_s3) && !env.CHANGE_ID) { + env.TAG = "v${version}" + log.info('########################## Checking if Artifact Already Exists. ###########################') + artifact_exists = sh(returnStdout: true, script: "gsutil ls \"gs://${env.objBucket}/${repo_name}/${branch_name}/${TAG}/\" 2>/dev/null || echo ''").trim() + log.info("${artifact_exists}") + if ( artifact_exists ) { + env.msg = "CI for this\n version - ${version}\n branch - ${branch_name}\n TAG - ${TAG}\nis already done. Please proceed with CD." + log.error(env.msg) + currentBuild.result = 'FAILURE' + throw new Exception(env.msg) + } + else { + log.info("Proceeding with building artifact for TAG - ${TAG}.") + } + } + else { + log.info('Skipping - Building artifact.') + } + } + } + } +} + +/* +Function to build the maven package +input arguments: +repo_name: repository name for changing dirctory +skip_test: boolen value to skip unit tests +skip_sonar: boolen value to skip sonar quality gate +args: string parameter to provide additional arguments to build cmd, example: '-U' +*/ +def build(String repo_name, boolean skip_test, String args) { + try { + if (env.CLOUD_PROVIDER == 'AWS') { + stage(stageName('Building gradle package')) { + String build_cmd = 'gradle clean build' + dir("$repo_name") { + sh(script:"cp ~/.m2/settings.xml .;${build_cmd} ${args}") + } + } + } + else if (env.CLOUD_PROVIDER == 'GCP') { + stage(stageName('Building gradle package')) { + String build_cmd = 'gradle clean build' + dir("$repo_name") { + sh(script:"gradle -v;cp ~/.m2/settings.xml .;${build_cmd} ${args}") + } + } + } + } + catch ( Exception e) { + env.msg = "Error Building the maven package. Please check console output for more details - ${e}" + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} +/* +Fuction to execute sonar scan + Input Arguments: + repo_name: repository name + skip_sonar: boolena parameter to skip sonar scan +*/ +def sonar_scan(String repo_name, boolean skip_sonar) { + try { + stage(stageName('Running sonar scan')) { + if (!skip_sonar) { + dir("$repo_name") { + withSonarQubeEnv(env.sonarEnv) { + if (env.CHANGE_ID) { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: "gradle sonar:sonar -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.repository=Meesho/${repo_name} -Dsonar.pullrequest.key=${env.CHANGE_ID} -Dsonar.pullrequest.branch=${env.CHANGE_BRANCH} -Dsonar.pullrequest.base=${env.CHANGE_TARGET}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script: "gradle sonar:sonar -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.repository=Meesho/${repo_name} -Dsonar.pullrequest.key=${env.CHANGE_ID} -Dsonar.pullrequest.branch=${env.CHANGE_BRANCH} -Dsonar.pullrequest.base=${env.CHANGE_TARGET}") + } + } + else { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: "gradle sonar:sonar -Dsonar.branch.name=${env.BRANCH_NAME}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script: "gradle sonar:sonar -Dsonar.branch.name=${env.BRANCH_NAME}") + } + } + } + } + } + else { + log.info('Skipping - Sonar Scan') + } + } + // if (!skip_sonar) { + // stage('Quality Gate') { + // timeout(time: 300, unit: 'SECONDS') { + // def qg = waitForQualityGate() + // catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + // if (qg.status != 'OK') { + // error "stage failed due to quality gate failure: ${qg.status}" + // } + // } + // } + // } + // } + } + catch (Exception e) { + dir("$repo_name") { + withSonarQubeEnv(env.sonarEnv) { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: 'gradle sonar:sonar') + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script: 'gradle sonar:sonar') + } + } + } + } +} +/* +Function to get the version from build.gradle +input arguments: +repo_name: String parameter to change the directory where build.gradle is located +*/ +def getVersion(String repo_name) { + try { + dir("$repo_name") { + return sh(returnStdout: true, script: "grep version build.gradle | head -1 | cut -d \"'\" -f2").trim() + } + } + catch ( Exception e) { + env.msg = 'Error while getting the version from build.gradle . Please check console output for more details.' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +/* +Function to get the modules from build.gradle +input arguments: +repo_name: String parameter to change the directory where build.gradle is located +*/ +def getModules(String repo_name) { + try { + dir("$repo_name") { + modules = sh(returnStdout: true, script: 'xq -r .project.modules.module[] build.gradle 2>/dev/null || xq -r .project.modules.module build.gradle 2>/dev/null || echo empty').trim() + if (modules == 'empty' || modules == 'null') { + modules = 'module_less' + } + modules = modules.split('\n') as List + return modules + } + } + catch ( Exception e) { + env.msg = 'Error getting the modules from build.gradle . Please check console output for more details.' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +/* +Function to push artifacts to jfrog artifactory +input arguments: +repo_name: repository name +push_to_jfrog: boolen argument to push_to_jfrog +*/ +def pushArtifactToJFrog(String repo_name, boolean push_to_jfrog) { + try { + stage(stageName('Deploying to JFrog')) { + if (env.CLOUD_PROVIDER == 'AWS') { + branch_name = 'repo' + if (branch_name == 'master' || branch_name == 'main' || push_to_jfrog) { + dir("${repo_name}") { + profiles = sh(returnStdout:true, script: 'xq -r .project.profiles build.gradle ').trim() + block_dist = sh(returnStdout:true, script: 'xq -r .project.distributionManagement build.gradle ').trim() + // Check for profiles tag in the build.gradle + if (profiles != 'null') { + // Check if profiles has distiributionManagement defined + profile_dist = sh(returnStdout:true, script: 'xq -r .project.profiles.profile[].distributionManagement build.gradle 2>/dev/null || xq -r .project.profiles.profile.distributionManagement build.gradle 2>/dev/null || echo null').trim() + if (profile_dist != 'null' ) { + publish_repo = (branch_name == 'master' || branch_name == 'main' || branch_name == 'gcp-main' || branch_name == 'gcp-master') ? 'useProdRepo' : 'useTestRepo' + log.info('########################### Pushing artifact to Jfrog. ###########################') + sh "gradle package deploy -DskipTests=true -D${publish_repo}=true" + } + else { + log.info('distributionManagement is not defined in the build.gradle . Skipping - Push to Jfrog Artifactory') + } + } + else if (block_dist != 'null') { + // Check if distributionManagement is defined without profiles + log.info('########################### Pushing artifact to Jfrog. ###########################') + sh 'gradle package deploy -DskipTests=true' + } + else { + log.info('distributionManagement is not defined in the build.gradle . Skipping - Push to Jfrog Artifactory') + } + } + } + else { + log.info('########################### Skipping - Push to Jfrog Artifactory. ###########################') + } + } + else if (env.CLOUD_PROVIDER == 'GCP') { + branch_name = 'repo' + if (branch_name == 'master' || branch_name == 'main' || push_to_jfrog) { + dir("${repo_name}") { + profiles = sh(returnStdout:true, script: 'xq -r .project.profiles build.gradle ').trim() + block_dist = sh(returnStdout:true, script: 'xq -r .project.distributionManagement build.gradle ').trim() + // Check for profiles tag in the build.gradle + if (profiles != 'null') { + // Check if profiles has distiributionManagement defined + profile_dist = sh(returnStdout:true, script: 'xq -r .project.profiles.profile[].distributionManagement build.gradle 2>/dev/null || xq -r .project.profiles.profile.distributionManagement build.gradle 2>/dev/null || echo null').trim() + if (profile_dist != 'null' ) { + publish_repo = (branch_name == 'master' || branch_name == 'main' || branch_name == 'gcp-main' || branch_name == 'gcp-master') ? 'useProdRepo' : 'useTestRepo' + log.info('########################### Pushing artifact to Jfrog. ###########################') + sh "gradle package deploy -DskipTests=true -D${publish_repo}=true" + echo 'execute gradle dependency' + sh 'gradle dependency:list > dependency_tree.txt' + echo ' uploading gradle dependency' + sh 'ls -ltr' + echo 'lets run copy command' + try { + sh "gsutil cp dependency_tree.txt 'gs://${env.objBucket}/common-dependencies/${repo_name}/'" + } + catch ( Exception e ) { + echo "Skipping - copying txt file to GCS - ${e}" + } + echo 'uploading gradle dependency' + } + else { + log.info('distributionManagement is not defined in the build.gradle . Skipping - Push to Jfrog Artifactory') + } + } + else if (block_dist != 'null') { + // Check if distributionManagement is defined without profiles + log.info('########################### Pushing artifact to Jfrog. ###########################') + sh 'gradle package deploy -DskipTests=true' + } + else { + log.info('distributionManagement is not defined in the build.gradle . Skipping - Push to Jfrog Artifactory') + } + } + } + else { + log.info('########################### Skipping - Push to Jfrog Artifactory. ###########################') + } + } + } + } + catch ( Exception e ) { + env.msg = 'Error in pushing artifacts to jfrog . Please check console output for more details.' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +/* +Function to push artifacts to s3 +input arguments: +repo_name: repository name +push_to_s3: boolen argument to push to s3 +*/ +def pushArtifactToS3(String repo_name, String branch_name, boolean push_to_s3) { + try { + stage(stageName('Pushing artifacts to object storage')) { + if (env.CLOUD_PROVIDER == 'AWS') { + if ("${branch_name}" == 'master' || "${branch_name}" == 'main' || push_to_s3) { + def modules = getModules("${repo_name}") + log.info('}########################### Pushing artifacts to S3. ###########################') + j = 0 + for (module in modules) { + j += 1 + sh """ + echo "${j}. ${module}" + if [ -f ${repo_name}/${module}/target/*.jar ] + then + ls -al ${repo_name}/${module}/target/*.jar + echo "Uploading artifacts to - ${repo_name}/${branch_name}/${TAG}" + aws s3 cp ${repo_name}/${module}/target/*.jar "s3://${env.objBucket}/${repo_name}/${branch_name}/${TAG}/${module}/" + echo "Listing ${TAG} artifacts -" + aws s3 ls "s3://${env.objBucket}/${repo_name}/${branch_name}/${TAG}/${module}/" || echo Nothing to display here. + elif [ -f ${repo_name}/target/*.jar ] && [ ${module} = 'module_less' ] + then + ls -al ${repo_name}/target/*.jar + echo "Uploading artifacts to - ${repo_name}/${branch_name}/${TAG}" + aws s3 cp ${repo_name}/target/*.jar "s3://${env.objBucket}/${repo_name}/${branch_name}/${TAG}/${module}/" + echo "Uploading Archived Code to - ${repo_name}/${branch_name}/${TAG}" + echo "Listing ${TAG} artifacts -" + aws s3 ls "s3://${env.objBucket}/${repo_name}/${branch_name}/${TAG}/${module}/" || echo Nothing to display here. + else + echo "No jar file found." + fi + """ + } + log.info("TAG for CD - ${TAG}") + } + else { + log.info("Skipping - Artifact push. As it is not supported for ${env.BRANCH_NAME}") + } + } + else if (env.CLOUD_PROVIDER == 'GCP') { + if ("${branch_name}" == 'master' || "${branch_name}" == 'main' || "${branch_name}" == 'gcp-main' || branch_name == 'gcp-master' || push_to_s3) { + def modules = getModules("${repo_name}") + log.info('}########################### Pushing artifacts to S3. ###########################') + j = 0 + for (module in modules) { + j += 1 + sh """ + echo "${j}. ${module}" + if [ -f ${repo_name}/${module}/target/*.jar ] + then + ls -al ${repo_name}/${module}/target/*.jar + echo "Uploading artifacts to - ${repo_name}/${branch_name}/${TAG}" + gsutil cp ${repo_name}/${module}/target/*.jar "gs://${env.objBucket}/${repo_name}/${branch_name}/${TAG}/${module}/" + echo "Listing ${TAG} artifacts -" + gsutil ls "gs://${env.objBucket}/${repo_name}/${branch_name}/${TAG}/${module}/" || echo Nothing to display here. + elif [ -f ${repo_name}/target/*.jar ] && [ ${module} = 'module_less' ] + then + ls -al ${repo_name}/target/*.jar + echo "Uploading artifacts to - ${repo_name}/${branch_name}/${TAG}" + gsutil cp ${repo_name}/target/*.jar "gs://${env.objBucket}/${repo_name}/${branch_name}/${TAG}/${module}/" + echo "Uploading Archived Code to - ${repo_name}/${branch_name}/${TAG}" + echo "Listing ${TAG} artifacts -" + gsutil ls "gs://${env.objBucket}/${repo_name}/${branch_name}/${TAG}/${module}/" || echo Nothing to display here. + else + echo "No jar file found." + fi + """ + } + log.info("TAG for CD - ${TAG}") + } + else { + log.info("Skipping - Artifact push. As it is not supported for ${env.BRANCH_NAME}") + } + } + } + } + catch ( Exception e ) { + env.msg = "Error in pushing artifacts to s3 bucket. Please check console output for more details - ${e}" + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +/* +Fuction for java version +*/ +def getDockerBuildVersion(String java_version) { + switch (java_version) { + case 'gradle': return '8-jdk-slim-secure-multiarch_v3.0' + } +} + +/* +Fuction to build maven docker repo +*/ + +def buildDckr(Map config) { + def btObj = new buTeamMapping() + def dparam_obj = new getDockerParams() + def constructObj = new constructTemplate() + + def team = btObj.get_team_initials(config.team) + def repo_name = config.repo_name + def tag = dparam_obj.getTag(repo_name) + def skip_test = config.skip_test ?: false + def modules = getModules("${repo_name}") + def excludedModules = config.excludedModules ?: [] + def deployArgo = config.deployArgo ?: false + def repoType = config.repo_type ?: 'microservice' + def docker_repo = "${env.cicd_environment}/${team}/${repo_name.toLowerCase()}" + env.JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64/' + env.PATH = "/opt/gradle/gradle-5.6.1/bin:${env.PATH}" + + def docker_bindings = [ + 'repo_name': repo_name, + 'buildRegistry': env.buildRegistry + ] + if (config.containsKey('copy_file')) { + // if (env.CLOUD_PROVIDER == "AWS"){ + // def recursive = config.copy_file.recursive ? ' --recursive' : '' + // } + // else if (env.CLOUD_PROVIDER == "GCP"){ + // def recursive = config.copy_file.recursive ? ' -r' : '' + // } + def recursive = config.copy_file.recursive ? ' --recursive' : '' + dir(repo_name) { + dir('copied_files') { + sh(script:"aws s3 cp${recursive} ${config.copy_file.path} .") + } + } + docker_bindings['copy_file'] = true + docker_bindings['copy_target'] = config.copy_file.target ?: '/opt/target' + } + else { + docker_bindings['copy_file'] = false + } + + if (modules != 'module_less') { + excludedModules.each { modules.removeElement(it) } + } + else { + modules = ['module_less'] + } + docker_bindings['arch'] = config.arch + def java_version = config.dockerBuildVersion + dockerBuildVersion = getDockerBuildVersion(java_version) + docker_bindings['dockerBuildVersion'] = dockerBuildVersion + run(config) + if (docker_bindings.copy_file) { + dir(repo_name) { + dir('copied_files') { + if (env.CLOUD_PROVIDER == 'AWS') { + // delete any old data + sh(script:'rm -rf *') + sh(script:"aws s3 cp${docker_bindings.recursive} ${docker_bindings.copy_file_path} .") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + // delete any old data + docker_bindings['recursive'] = '-r' + sh(script:'rm -rf *') + sh(script:"gsutil cp ${docker_bindings.recursive} ${docker_bindings.copy_file_path}* .") + } + } + } + } + stage(stageName('Building docker images')) { + // Login to docker + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws ecr get-login-password --region ${env.region} | docker login --username AWS --password-stdin ${env.registry}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script:'gcloud auth configure-docker asia-southeast1-docker.pkg.dev --quiet') + } + dir(repo_name) { + for (module in modules) { + docker_bindings['module'] = module + constructObj.renderTemplate(docker_bindings,'java-Dockerfile','Dockerfile-' + module) + def module_repo = (module == 'module_less') ? docker_repo : docker_repo + '/' + module + if ( module == 'module_less') { + sh(script: 'mkdir target;cp build/libs/*jar target') + } + else { + sh(script: "mkdir -p ${module}/target;cp build/libs/*jar ${module}/target") + } + if (env.cicd_environment != 'ftr' && repoType == 'microservice') { + sh(script: "ls target; docker build --tag ${env.registry}/${module_repo}:${tag} -f Dockerfile-${module} . && docker push ${env.registry}/${module_repo}:${tag}") + } + else { + log.info("Skipping Docker Build - ${env.cicd_environment} env") + log.info("Skipping Docker Build - ${repoType} repo type") + } + //Remove dockerfile + sh(script: "rm -rf Dockerfile-${module}") + } + } + return [tag , deployArgo] + } +} diff --git a/src/com/meesho/stages/buildMaven.groovy b/src/com/meesho/stages/buildMaven.groovy new file mode 100644 index 0000000..cf190c2 --- /dev/null +++ b/src/com/meesho/stages/buildMaven.groovy @@ -0,0 +1,1059 @@ +package com.meesho.stages + +import com.meesho.utilities.buTeamMapping +import com.meesho.utilities.constructTemplate +import com.meesho.utilities.getDockerParams +import com.meesho.stages.checkOut +import com.meesho.utilities.gitActions +import com.meesho.utilities.constructParam +import com.meesho.utilities.dockerUtilities +/* +Function to define the flow of entire build, this function will call different stages related to maven build +*/ +def run(Map config) { + // get required variables from config + def checkoutObj = new checkOut() + def repo_name = config.repo_name + def args = config.build_args ?: '' + def skip_test = config.skip_test ?: false + def skip_sonar = config.skip_sonar ?: false + def push_to_jfrog = config.push_to_jfrog ?: false + def push_to_s3 = config.push_to_s3 ?: false + def skip_s3_check = config.skip_s3_check ?: false + def deployArgo = config.deployArgo ?: false + def branch_name = "${env.BRANCH_NAME}" + def appConfigChanges = false + def continue_build = true + def repoType = config.repo_type ?: 'microservice' + + if (branch_name == 'gcp-main' || branch_name == 'gcp-master') { + // push_to_jfrog = false + push_to_jfrog = config.containsKey('push_to_jfrog') ? config.push_to_jfrog : true + } + def version = getVersion("${repo_name}") + env.TAG = "v${version}" + def java_version = config.dockerBuildVersion + + echo "${env.req_java} ${java_version}" + + if (env.INFRA_ENV == 'toolchain') { + skip_s3_check = true + skip_sonar = true + push_to_jfrog = false + push_to_s3 = false + } + + if (env.hot_fix) { + skip_test = true + skip_sonar = true + push_to_jfrog = false + } + + if (config.enable_submodule) { + checkoutObj.chekoutSubmodule(repo_name) + } + // call the stages + echo "Artifact Check - ${skip_s3_check}" + if (env.hot_fix) { + env.TAG = "v${version}-HOT" + } + else if (skip_s3_check) { + log.info('Skipping - Checking if artifact already exists.') + } + else { + echo 'Checking if artifact already exists.' + continue_build = checkS3(repo_name, branch_name, env.objBucket, version, push_to_s3) + } + + + if (continue_build) { + //meanwhile latest commit check is not fixed creating build for appConfig changes also + (appConfigChanges , deployArgo) = AppConfigChangeCheck(repo_name, deployArgo.toBoolean()) + log.info("Is just application config change ? $appConfigChanges") + + //if appConfigChanges then skip the build and artifact push just run quality gate + if (!appConfigChanges){ + echo "proceeding with the build" + build(repo_name, skip_test, args,repoType) + //check for mq version + checkMQVersion(repo_name) + } + + sonar_scan(repo_name, skip_sonar,appConfigChanges) + + if (!appConfigChanges){ + if (!env.CHANGE_ID) { + pushArtifactToJFrog(repo_name, push_to_jfrog) + pushArtifactToS3(repo_name, branch_name, env.objBucket, push_to_s3) + // Set a flag to indicate S3 push was successful for cleanup purposes + env.S3_PUSH_SUCCESS = 'true' + } + else { + log.info('Artifact Push is disabled for Pull Requests') + } + } + continue_build = !appConfigChanges + } + return [continue_build , deployArgo] +} + +/* +Function to check if the artifact exist in s3 or not +input argument +*/ +def checkS3(String repo_name, String branch_name, String bucket_name, String version, boolean push_to_s3) { + stage(stageName('Checking if Jar already exists in GCS for current version')) { + if (env.CLOUD_PROVIDER == 'AWS') { + if (("${branch_name}" == 'master' || "${branch_name}" == 'main' || push_to_s3) && !env.CHANGE_ID) { + log.info('########################## Checking if Artifact Already Exists. ###########################') + artifact_exists = sh(returnStdout: true, script: "aws s3 ls \"s3://${env.objBucket}/${repo_name}/${branch_name}/${env.TAG}/\" 2>/dev/null || echo ''").trim() + log.info("${artifact_exists}") + if ( artifact_exists ) { + env.msg = "CI for this\n version - ${version}\n branch - ${branch_name}\n TAG - ${env.TAG}\nis already done. Please proceed with CD." + log.error(env.msg) + return false + } + else { + log.info("Proceeding with building artifact for TAG - ${env.TAG}.") + } + } + else { + log.info('Skipping - Building artifact') + } + return true + } + else if (env.CLOUD_PROVIDER == 'GCP') { + if (("${branch_name}" == 'master' || "${branch_name}" == 'main' || "${branch_name}" == 'gcp-main' || "${branch_name}" == 'gcp-master' || push_to_s3) && !env.CHANGE_ID) { + log.info('########################## Checking if Artifact Already Exists. ###########################') + artifact_exists = sh(returnStdout: true, script: "gsutil ls \"gs://${env.objBucket}/${repo_name}/${branch_name}/${env.TAG}/\" 2>/dev/null || echo ''").trim() + log.info("${artifact_exists}") + if ( artifact_exists ) { + env.msg = "CI for this\n version - ${version}\n branch - ${branch_name}\n TAG - ${env.TAG}\nis already done. Please proceed with CD." + log.error(env.msg) + return false + } + else { + log.info("Proceeding with building artifact for TAG - ${env.TAG}.") + } + } + else { + log.info('Skipping - Building artifact.') + } + return true + } + } +} + + +//check the application Config Change in github +//true when only changes are present in configs dir of repo +def AppConfigChangeCheck(String repo_name, boolean deployArgo){ + if (env.INFRA_ENV == 'toolchain') { + return [false , deployArgo] + } + def gitObj = new gitActions() + def configFiles = [] + def appConfigChanges = false + // Get changed files + def changedFiles = env.CHANGE_ID ? + gitObj.fetchDiffFilesForPullRequest(repo_name, env.CHANGE_TARGET) : + gitObj.fetchDiffFilesForPushRequest(repo_name) + + if (!changedFiles) { + return [appConfigChanges , deployArgo] + } + + // Process changed files + changedFiles.split('\n').each { file -> + if (!file.startsWith('configs/')) { + appConfigChanges = false + return [appConfigChanges , deployArgo] // Return from function if any non-config file + } + appConfigChanges = true + configFiles.add(file) + } + + // If we only have config files, check if they contain cicd_environment + if (configFiles && deployArgo && appConfigChanges) { + def hasEnvironmentConfig = configFiles.any { it.contains(env.cicd_environment) } + // Only set should_auto_cd to false if no config contains cicd_environment + if (!hasEnvironmentConfig) { + deployArgo = false + echo "Config changes don't contain environment: ${env.cicd_environment}" + } + } + return [appConfigChanges , deployArgo] +} + + +/* +Function to build the maven package +input arguments: +repo_name: repository name for changing dirctory +skip_test: boolen value to skip unit tests +skip_sonar: boolen value to skip sonar quality gate +args: string parameter to provide additional arguments to build cmd, example: '-U' +*/ +def build(String repo_name, boolean skip_test, String args, String repoType) { + def validation_output = "" + try { + stage(stageName('Running CaC validation script')) { + def param = new constructParam() + if (env.CHANGE_ID && repoType == 'microservice') { + def ValidateConfig = param.ValidateCacConfigForRepo(false,repo_name) + if (ValidateConfig) { + log.info('************ Validate Config for CAC application.yml files ************') + dir("$repo_name") { + writeFile file: 'validate_configs.py', text: libraryResource('com/meesho/validate_configs.py') + //Capture output separately, then check status + validation_output = sh( + script: 'python3 validate_configs.py 2>&1 || true', // || true prevents immediate failure + returnStdout: true + ).trim() + //Check if validation actually failed by examining output or running again + def exit_status = sh( + script: 'python3 validate_configs.py 2>/dev/null', + returnStatus: true + ) + + if (exit_status != 0) { + // If the script failed, manually throw an exception to enter the catch block + throw new Exception("Config validation failed with exit code ${exit_status}. See output for details.") + } + } + } else { + log.info('************ Skipping Validation of CAC Config ************') + } + } else { + log.info("Skipping Validation of CAC Config for ${repoType} repo type") + log.info('************ Skipping Validation of CAC Config ************') + } + } + + if (env.CLOUD_PROVIDER == 'AWS') { + stage(stageName('Building maven package')) { + String build_cmd = 'mvn clean install' + if (skip_test) { + args += ' -DskipTests' + } + dir("$repo_name") { + sh(script:"cp ~/.m2/settings.xml .;${build_cmd} ${args}") + } + } + } + else if (env.CLOUD_PROVIDER == 'GCP') { + stage(stageName('Building maven package')) { + String build_cmd = "JAVA_HOME=${env.req_java} mvn clean install" + if (skip_test) { + args = args + ' -DskipTests' + } + dir("$repo_name") { + full_script = "JAVA_HOME=${env.req_java} mvn -v; cp ~/.m2/settings.xml .; ${build_cmd} ${args}" + + def result = runCommandWithOutput(full_script) + + def maven_build_output = result.output + def exit_status = result.status + echo '$exit_status' + if (exit_status != 0) { + throw new Exception("Maven build failed with exit code ${exit_status}. See output for details.") + } + } + } + } + } + catch ( Exception e) { + def is_validation_error = !validation_output.isEmpty() && !validation_output.contains("Validation successful.") + def has_maven_build_output = false + def detailed_error_msg = has_maven_build_output && !is_validation_error ? + "--- Maven Build Failure Details ---\n" + maven_build_output : + (is_validation_error ? + validation_output : + e.toString()) + env.msg = "Error Building the maven package. If validate script error then please refer to doc - https://meesho.atlassian.net/wiki/spaces/EW/pages/3915972744/Config+Schema+Validation+Common+Errors+And+Fixes . Also Please check console output for more details - ${e}" + env.error_msg_to_db = detailed_error_msg + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} +/* +Fuction to execute sonar scan + Input Arguments: + repo_name: repository name + skip_sonar: boolena parameter to skip sonar scan +*/ +def sonar_scan(String repo_name, boolean skip_sonar , boolean appConfigChanges) { + try { + stage(stageName('Running sonar scan')) { + if (!skip_sonar && !appConfigChanges) { + dir("$repo_name") { + withSonarQubeEnv(env.sonarEnv) { + if (env.CHANGE_ID) { + if (env.CLOUD_PROVIDER == 'AWS') { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh "git fetch origin ${env.CHANGE_TARGET}:refs/remotes/origin/${env.CHANGE_TARGET}" + } + sh(script: "mvn sonar:sonar -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.repository=Meesho/${repo_name} -Dsonar.pullrequest.key=${env.CHANGE_ID} -Dsonar.pullrequest.branch=${env.CHANGE_BRANCH} -Dsonar.pullrequest.base=${env.CHANGE_TARGET} -Dsonar.ws.timeout=120") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh "git fetch origin ${env.CHANGE_TARGET}:refs/remotes/origin/${env.CHANGE_TARGET}" + } + sh(script: "mvn sonar:sonar -Dsonar.pullrequest.provider=GitHub -Dsonar.projectName=${repo_name} -Dsonar.pullrequest.github.repository=Meesho/${repo_name} -Dsonar.pullrequest.key=${env.CHANGE_ID} -Dsonar.pullrequest.branch=${env.CHANGE_BRANCH} -Dsonar.pullrequest.base=${env.CHANGE_TARGET} -Dsonar.ws.timeout=120") + } + } + else { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: "mvn sonar:sonar -Dsonar.branch.name=${env.BRANCH_NAME} -Dsonar.ws.timeout=120") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script: "mvn sonar:sonar -Dsonar.branch.name=${env.BRANCH_NAME} -Dsonar.projectName=${repo_name} -Dsonar.ws.timeout=120") + } + } + } + } + } + else { + log.info('Skipping - Sonar Scan') + } + } + stage(stageName('Quality Gate')){ + if (appConfigChanges){ + log.info("Skipping the quality gate check as Application config change ") + } + else if (skip_sonar){ + log.info("Sonar scan is skipped. Marking this stage as passed.") + } + else if (!env.CHANGE_ID || env.cicd_environment == 'int'){ + log.info("Skipping quality gate check on Branches/Pre-Prod. Marking this stage as passed.") + } + else{ + def timeoutValue = (env.INFRA_ENV == 'prd') ? 600 : 360 + timeout(time: timeoutValue, unit: 'SECONDS') { + def qg = waitForQualityGate() + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE'){ + if (qg.status != 'OK') { + error "stage failed due to quality gate failure: ${qg.status}" + } + } + } + } + } + } + catch (Exception e) { + dir("$repo_name") { + withSonarQubeEnv(env.sonarEnv) { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: 'mvn sonar:sonar -Dsonar.ws.timeout=120') + } + else if (env.CLOUD_PROVIDER == 'GCP') { + log.info("Sonar Scan or Quality Gate has failed. Continuing with the build.") + env.error_msg_to_db = 'Sonar Scan or Quality Gate has failed. Continuing with the build.' + env.msg = 'Sonar Scan or Quality Gate has failed. Continuing with the build.' + } + } + } + } +} +/* +Function to get the version from pom.xml +input arguments: +repo_name: String parameter to change the directory where pom.xml is located +*/ +def getVersion(String repo_name) { + try { + dir("$repo_name") { + def pom_version = sh(returnStdout: true, script: 'xq -r .project.version pom.xml').trim() + if (pom_version.contains('null')) { + env.msg = 'Error while getting the version from pom.xml. Please check console output for more details.' + env.error_msg_to_db = 'Error Getting the version from pom.xml' + log.error(env.msg) + currentBuild.result = env.FAILURE + sh 'exit 1' + } + else { + return pom_version + } + } + } + catch ( Exception e) { + env.msg = 'Error while getting the version from pom.xml. Please check console output for more details.' + env.error_msg_to_db = 'Error Getting the version from pom.xml' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +/* +Function to get the modules from pom.xml +input arguments: +repo_name: String parameter to change the directory where pom.xml is located +*/ +def getModules(String repo_name) { + try { + dir("$repo_name") { + modules = sh(returnStdout: true, script: 'xq -r .project.modules.module[] pom.xml 2>/dev/null || xq -r .project.modules.module pom.xml 2>/dev/null || echo empty').trim() + if (modules == 'empty' || modules == 'null') { + modules = 'module_less' + } + modules = modules.split('\n') as List + return modules + } + } + catch ( Exception e) { + env.msg = 'Error getting the modules from pom.xml. Please check console output for more details.' + env.error_msg_to_db = 'Error getting the modules from pom.xml' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +/* +Function to push artifacts to jfrog artifactory +input arguments: +repo_name: repository name +push_to_jfrog: boolen argument to push_to_jfrog +*/ +def pushArtifactToJFrog(String repo_name, boolean push_to_jfrog) { + try { + stage(stageName('Deploying to JFrog')) { + if (env.CLOUD_PROVIDER == 'AWS') { + branch_name = "${env.BRANCH_NAME}" + if (branch_name == 'master' || branch_name == 'main' || push_to_jfrog) { + dir("${repo_name}") { + profiles = sh(returnStdout:true, script: 'xq -r .project.profiles pom.xml').trim() + block_dist = sh(returnStdout:true, script: 'xq -r .project.distributionManagement pom.xml').trim() + // Check for profiles tag in the pom.xml + if (profiles != 'null') { + // Check if profiles has distiributionManagement defined + profile_dist = sh(returnStdout:true, script: 'xq -r .project.profiles.profile[].distributionManagement pom.xml 2>/dev/null || xq -r .project.profiles.profile.distributionManagement pom.xml 2>/dev/null || echo null').trim() + if (profile_dist != 'null' ) { + publish_repo = (branch_name == 'master' || branch_name == 'main' || branch_name == 'gcp-main' || branch_name == 'gcp-master') ? 'useProdRepo' : 'useTestRepo' + log.info('########################### Pushing artifact to Jfrog. ###########################') + sh "mvn package deploy -DskipTests=true -D${publish_repo}=true" + } + else { + log.info('distributionManagement is not defined in the pom.xml. Skipping - Push to Jfrog Artifactory') + } + } + else if (block_dist != 'null') { + // Check if distributionManagement is defined without profiles + log.info('########################### Pushing artifact to Jfrog. ###########################') + sh 'mvn package deploy -DskipTests=true' + } + else { + log.info('distributionManagement is not defined in the pom.xml. Skipping - Push to Jfrog Artifactory') + } + } + } + else { + log.info('########################### Skipping - Push to Jfrog Artifactory. ###########################') + } + } + else if (env.CLOUD_PROVIDER == 'GCP') { + branch_name = "${env.BRANCH_NAME}" + if (branch_name == 'master' || branch_name == 'main' || push_to_jfrog) { + dir("${repo_name}") { + profiles = sh(returnStdout:true, script: 'xq -r .project.profiles pom.xml').trim() + block_dist = sh(returnStdout:true, script: 'xq -r .project.distributionManagement pom.xml').trim() + // Check for profiles tag in the pom.xml + if (profiles != 'null') { + // Check if profiles has distiributionManagement defined + profile_dist = sh(returnStdout:true, script: 'xq -r .project.profiles.profile[].distributionManagement pom.xml 2>/dev/null || xq -r .project.profiles.profile.distributionManagement pom.xml 2>/dev/null || echo null').trim() + if (profile_dist != 'null' ) { + publish_repo = (branch_name == 'master' || branch_name == 'main' || branch_name == 'gcp-main' || branch_name == 'gcp-master') ? 'useProdRepo' : 'useTestRepo' + log.info('########################### Pushing artifact to Jfrog. ###########################') + sh "JAVA_HOME=${env.req_java} mvn package deploy -DskipTests=true -D${publish_repo}=true" + } + else { + log.info('distributionManagement is not defined in the pom.xml. Skipping - Push to Jfrog Artifactory') + } + } + else if (block_dist != 'null') { + // Check if distributionManagement is defined without profiles + log.info('########################### Pushing artifact to Jfrog. ###########################') + sh "JAVA_HOME=${env.req_java} mvn package deploy -DskipTests=true" + } + else { + log.info('distributionManagement is not defined in the pom.xml. Skipping - Push to Jfrog Artifactory') + } + } + } + else { + log.info('########################### Skipping - Push to Jfrog Artifactory. ###########################') + } + } + } + } + catch ( Exception e ) { + env.msg = 'Error in pushing artifacts to jfrog . Please check console output for more details.' + env.error_msg_to_db = 'Error in Pushing Artifacts To Jfrog' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +/* +Function to push artifacts to s3 +input arguments: +repo_name: repository name +push_to_s3: boolen argument to push to s3 +*/ +def pushArtifactToS3(String repo_name, String branch_name, String bucket_name, boolean push_to_s3) { + try { + stage(stageName('Pushing artifacts to GCS bucket')) { + if (env.CLOUD_PROVIDER == 'AWS') { + if ("${branch_name}" == 'master' || "${branch_name}" == 'main' || push_to_s3) { + def modules = getModules("${repo_name}") + log.info('}########################### Pushing artifacts to S3. ###########################') + j = 0 + for (module in modules) { + j += 1 + sh """ + echo "${j}. ${module}" + if [ -f ${repo_name}/${module}/target/*.jar ] + then + ls -al ${repo_name}/${module}/target/*.jar + echo "Uploading artifacts to - ${repo_name}/${branch_name}/${env.TAG}" + aws s3 cp ${repo_name}/${module}/target/*.jar "s3://${env.objBucket}/${repo_name}/${branch_name}/${env.TAG}/${module}/" + echo "Listing ${env.TAG} artifacts -" + aws s3 ls "s3://${env.objBucket}/${repo_name}/${branch_name}/${env.TAG}/${module}/" || echo Nothing to display here. + elif [ -f ${repo_name}/target/*.jar ] && [ ${module} = 'module_less' ] + then + ls -al ${repo_name}/target/*.jar + echo "Uploading artifacts to - ${repo_name}/${branch_name}/${env.TAG}" + aws s3 cp ${repo_name}/target/*.jar "s3://${env.objBucket}/${repo_name}/${branch_name}/${env.TAG}/${module}/" + echo "Uploading Archived Code to - ${repo_name}/${branch_name}/${env.TAG}" + echo "Listing ${env.TAG} artifacts -" + aws s3 ls "s3://${env.objBucket}/${repo_name}/${branch_name}/${env.TAG}/${module}/" || echo Nothing to display here. + else + echo "No jar file found." + fi + """ + } + log.info("TAG for CD - ${env.TAG}") + } + else { + log.info("Skipping - Artifact push. As it is not supported for ${env.BRANCH_NAME}") + } + } + else if (env.CLOUD_PROVIDER == 'GCP') { + if ("${branch_name}" == 'master' || "${branch_name}" == 'main' || "${branch_name}" == 'gcp-main' || branch_name == 'gcp-master' || push_to_s3) { + def modules = getModules("${repo_name}") + log.info('}########################### Pushing artifacts to S3. ###########################') + j = 0 + for (module in modules) { + j += 1 + sh """ + echo "${j}. ${module}" + if [ -f ${repo_name}/${module}/target/*.jar ] + then + ls -al ${repo_name}/${module}/target/*.jar + echo "Uploading artifacts to - ${repo_name}/${branch_name}/${env.TAG}" + gsutil cp ${repo_name}/${module}/target/*.jar "gs://${env.objBucket}/${repo_name}/${branch_name}/${env.TAG}/${module}/" + echo "Listing ${env.TAG} artifacts -" + gsutil ls "gs://${env.objBucket}/${repo_name}/${branch_name}/${env.TAG}/${module}/" || echo Nothing to display here. + elif [ -f ${repo_name}/target/*.jar ] && [ ${module} = 'module_less' ] + then + ls -al ${repo_name}/target/*.jar + echo "Uploading artifacts to - ${repo_name}/${branch_name}/${env.TAG}" + gsutil cp ${repo_name}/target/*.jar "gs://${env.objBucket}/${repo_name}/${branch_name}/${env.TAG}/${module}/" + echo "Uploading Archived Code to - ${repo_name}/${branch_name}/${env.TAG}" + echo "Listing ${env.TAG} artifacts -" + gsutil ls "gs://${env.objBucket}/${repo_name}/${branch_name}/${env.TAG}/${module}/" || echo Nothing to display here. + else + echo "No jar file found." + fi + """ + } + log.info("TAG for CD - ${env.TAG}") + } + else { + log.info("Skipping - Artifact push. As it is not supported for ${env.BRANCH_NAME}") + } + } + } + } + catch ( Exception e ) { + env.msg = "Error in pushing artifacts to s3 bucket. Please check console output for more details - ${e}" + env.error_msg_to_db = 'Error in Pushing Artifacts to S3 Buckets' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +def stageDocker(def repo_name, def region, def registry, def docker_repo, def docker_bindings, def modules, def tag, def tagShort, def build_docker, def team) { + def constructObj = new constructTemplate() + def dockerUtilObj = new dockerUtilities() + if (docker_bindings.copy_file) { + dir(repo_name) { + dir('copied_files') { + if (env.CLOUD_PROVIDER == 'AWS') { + // delete any old data + sh(script:'rm -rf *') + sh(script:"aws s3 cp${docker_bindings.recursive} ${docker_bindings.copy_file_path} .") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + // delete any old data + docker_bindings['recursive'] = '-r' + sh(script:'rm -rf *') + sh(script:"gsutil cp ${docker_bindings.recursive} ${docker_bindings.copy_file_path}* .") + } + } + } + } + stage(stageName('Building docker images')) { + if (build_docker) { + // Login to docker + try { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws ecr get-login-password --region ${env.region} | docker login --username AWS --password-stdin ${env.registry}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script:'gcloud auth configure-docker asia-southeast1-docker.pkg.dev --quiet') + } + } + catch (Exception e) { + env.msg = 'Error in Docker login' + env.error_msg_to_db = 'Error in Docker login' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + dir(repo_name) { + try { + for (module in modules) { + docker_bindings['module'] = module + constructObj.renderTemplate(docker_bindings, 'java-Dockerfile', 'Dockerfile-' + module) + module_repo = (module == 'module_less') ? docker_repo : docker_repo + '/' + module + // Check and craete ECR + try { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws ecr describe-repositories --region ${env.region} --repository-names ${module_repo} || aws ecr create-repository --region ${env.region} --repository-name ${module_repo}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + echo 'Skipping - Registry Creation in GCP.' + } + } + catch (Exception e) { + env.msg = "Error in creating ECR repository ${module_repo}" + env.error_msg_to_db = "Error in creating ECR repository ${module_repo}" + currentBuild.result = env.FAILURE + throw e + } + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: "docker buildx build --platform linux/arm64,linux/amd64 --tag ${env.registry}/${module_repo}:${tag} -f Dockerfile-${module} --push .") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + // sleep(10000) + sh(script: "docker build --tag ${env.registry}/${module_repo}:${tag} -f Dockerfile-${module} .") + if (env.cicd_environment != 'ftr' || env.INFRA_ENV == 'toolchain') { + dockerUtilObj.retryDockerPush("docker push ${env.registry}/${module_repo}:${tag}") + } else { + log.info("Skipping Docker Push - ${env.cicd_environment} env") + } + } + + //Remove dockerfile + sh(script: "rm -rf Dockerfile-${module}") + } + } + catch (Exception e) { + env.msg = 'Error in building DockerFile Or Pushing To ECR' + env.error_msg_to_db = env.msg + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + } + } + else { + log.info('Getting latest image tag from ECR') + mod_docker_repo = (modules[0] == 'module_less') ? docker_repo : docker_repo + '/' + modules[0] + try { + if (env.CLOUD_PROVIDER == 'AWS') { + tag = sh(script: "aws ecr describe-images --output json --repository-name ${mod_docker_repo} --query 'sort_by(imageDetails,& imagePushedAt)[].imageTags[]|[-1]'", returnStdout: true).replaceAll("[\n\"]", '') + } + else if (env.CLOUD_PROVIDER == 'GCP') { + latest_image = sh(script: "gcloud container images list-tags ${env.registry}/${mod_docker_repo} --format='value(tags)' | sed '/^\$/d' | awk -F'-' '{print \$NF}' | sort | tail -1", returnStdout: true).trim() + tag = sh(script: "gcloud container images list-tags ${env.registry}/${mod_docker_repo} --format='value(tags)' | grep ${latest_image}", returnStdout: true).trim() + // tag = sh(script: "gcloud container images list-tags ${env.registry}/${mod_docker_repo} --format='value(tags)' | grep -i '${tagShort}' | sort | tail -1", returnStdout: true).trim() + } + } + catch (Exception e) { + env.msg = 'Error getting latest image from ECR' + env.error_msg_to_db = env.msg + log.error(env.msg + '\n' + e.toString()) + currentBuild.result = env.FAILURE + throw e + } + } + return tag + } +} + +def getDockerBuildVersion(String java_version) { + switch (java_version) { + case 'maven-graalvm': return 'slim-jdk8-graalvm' + case 'maven-graalvm-11': return '11-jdk-slim-graalvm' + case 'maven-3.3-jdk-11': return '11-java' + case 'maven-3.3-jdk-8': return '8-jdk-slim-secure-multiarch_v3.0' + case 'maven-3.3-jdk-17': return '17-jdk-slim-secure-multiarch_v2.0' + case 'maven-3.9-jdk-25': return '25-jdk-noble-amd64' + } +} + +def getReqJavaPath(String java_version) { + switch (java_version) { + case 'maven-3.9-jdk-25': return '/usr/lib/jvm/java-25-openjdk-amd64/' + case 'maven-3.3-jdk-17': return '/usr/lib/jvm/java-17-openjdk-amd64/' + case 'maven-3.3-jdk-11': return '/usr/lib/jvm/java-17-openjdk-amd64/' + case 'maven-graalvm': return '/usr/lib/jvm/java-8-openjdk-amd64/' + case 'maven-graalvm-11': return '/usr/lib/jvm/java-17-openjdk-amd64/' + default: return '/usr/lib/jvm/java-8-openjdk-amd64/' + } +} +/* +Fuction to build maven docker repo +*/ + +def buildDckr(Map config) { + def btObj = new buTeamMapping() + def dparam_obj = new getDockerParams() + + def team = btObj.get_team_initials(config.team) + def repo_name = config.repo_name + def skip_test = config.skip_test ?: false + def modules = getModules("${repo_name}") + def excludedModules = config.excludedModules ?: [] + def docker_repo = "${env.cicd_environment}/${team}/${repo_name.toLowerCase()}" + def module_repo = '' + def tag = dparam_obj.getTag(repo_name) + def tagShort = dparam_obj.getTagShort(repo_name) + def deployArgo = config.deployArgo ?: false + def build_docker + def java_version = config.dockerBuildVersion + def repoType = config.repo_type ?: 'microservice' + env.req_java = '/usr/lib/jvm/java-8-openjdk-amd64/' + boolean allImagesExist = true + if (env.INFRA_ENV == 'toolchain') { + def dockerUtilObj = new dockerUtilities() + for (module in modules) { + def moduleName = (module instanceof LinkedHashMap) ? module.keySet()[0] : module + def modulePath = (moduleName == 'module_less') ? docker_repo : "${docker_repo}/${moduleName}" + if (excludedModules.contains(moduleName)) { + continue + } + if (!dockerUtilObj.imageExists(env.registry, modulePath, tag)) { + log.info("Toolchain: Image missing for ${moduleName} at ${modulePath}:${tag}. Proceeding with build.") + allImagesExist = false + break + } + } + if (allImagesExist) { + log.info("Toolchain: All images found in registry for tag ${tag}. Skipping build step.") + return [tag, deployArgo] + } + } + + def docker_bindings = [ + 'repo_name': repo_name, + 'buildRegistry': env.buildRegistry, + 'CLOUD_PROVIDER': env.CLOUD_PROVIDER + ] + if (config.containsKey('copy_file')) { + // if (env.CLOUD_PROVIDER == "AWS"){ + // def recursive = config.copy_file.recursive ? ' --recursive' : '' + // } + // else if (env.CLOUD_PROVIDER == "GCP"){ + // def recursive = config.copy_file.recursive ? ' -r' : '' + // } + def recursive = config.copy_file.recursive ? ' --recursive' : '' + docker_bindings['copy_file'] = true + docker_bindings['recursive'] = recursive + docker_bindings['copy_file_path'] = config.copy_file.path + docker_bindings['copy_target'] = config.copy_file.target ?: '/opt/target' + } + else { + docker_bindings['copy_file'] = false + } + + if (modules != 'module_less') { + excludedModules.each { modules.removeElement(it) } + } + else { + modules = ['module_less'] + } + docker_bindings['arch'] = config.arch + def dockerBuildVersion = '' + + + dockerBuildVersion = getDockerBuildVersion(java_version) + docker_bindings['dockerBuildVersion'] = dockerBuildVersion + if (java_version != 'maven-3.3-jdk-8' && java_version != 'maven-graalvm') { + try { + if (env.CLOUD_PROVIDER == 'AWS') { + node(java_version) { + def checkObj = new checkOut() + checkObj.run(['repo_name': repo_name]) + (build_docker , deployArgo) = run(config) + tag = stageDocker(repo_name, env.region, env.registry, docker_repo, docker_bindings, modules, tag, tagShort, build_docker, team) + } + } + else if (env.CLOUD_PROVIDER == 'GCP') { + + env.req_java = getReqJavaPath(java_version) + echo "CICD - Using JAVA_HOME from config.dockerBuildVersion (${java_version}): ${env.req_java}" + // sleep(32764572) + + (build_docker , deployArgo) = run(config) + if ((env.cicd_environment != 'ftr' || env.INFRA_ENV == 'toolchain') && repoType == 'microservice') { + log.info("1. Building Docker image for ${modules}") + tag = stageDocker(repo_name, env.region, env.registry, docker_repo, docker_bindings, modules, tag, tagShort, build_docker, team) + } + else { + log.info("1 Skipping Docker build for ${env.cicd_environment} env") + log.info("1 Skipping Docker build for ${repoType} repo type") + } + } + // If we reach here, the entire pipeline succeeded + env.PIPELINE_SUCCESS = 'true' + } catch (Exception pipelineException) { + // Any exception - handled or unhandled - will reach here if re-thrown + throw pipelineException + } finally { + // Cleanup if S3 succeeded but pipeline didn't complete successfully + if (env.S3_PUSH_SUCCESS == 'true' && env.PIPELINE_SUCCESS != 'true') { + log.error("Pipeline did not complete successfully after S3 push. Cleaning up S3 artifacts.") + cleanupGCSArtifacts(repo_name, "${env.BRANCH_NAME}", env.objBucket) + } + } + } + else { + try { + (build_docker , deployArgo) = run(config) + if ((env.cicd_environment != 'ftr' || env.INFRA_ENV == 'toolchain') && repoType == 'microservice' ) { + tag = stageDocker(repo_name, env.region, env.registry, docker_repo, docker_bindings, modules, tag, tagShort, build_docker, team) + } + else { + log.info("2 Skipping Docker build for ${env.cicd_environment} env") + log.info("2 Skipping Docker build for ${repoType} repo type") + } + // If we reach here, the entire pipeline succeeded + env.PIPELINE_SUCCESS = 'true' + } catch (Exception pipelineException) { + // Any exception - handled or unhandled - will reach here if re-thrown + throw pipelineException + } finally { + // Cleanup if S3 succeeded but pipeline didn't complete successfully + if (env.S3_PUSH_SUCCESS == 'true' && env.PIPELINE_SUCCESS != 'true') { + log.error("Pipeline did not complete successfully after S3 push. Cleaning up S3 artifacts.") + cleanupGCSArtifacts(repo_name, "${env.BRANCH_NAME}", env.objBucket) + } + } + } + return [tag , deployArgo] +} + +def checkMQVersion(String repo_name) { + try { + if (env.CLOUD_PROVIDER == 'AWS') { + stage(stageName('Building maven package')) { + String build_cmd = 'mvn clean install' + if (skip_test) { + args += ' -DskipTests' + } + dir("$repo_name") { + sh(script:"cp ~/.m2/settings.xml .;${build_cmd} ${args}") + } + } + } + else if (env.CLOUD_PROVIDER == 'GCP') { + stage(stageName('Checking MQ version dependency')) { + String build_cmd = "JAVA_HOME=${env.req_java} mvn dependency:tree" + dir("$repo_name") { + sh "pwd" + sh(script: "JAVA_HOME=${env.req_java} mvn -v; cp ~/.m2/settings.xml .; ${build_cmd} > dependencies.txt") + log.info("dependency tree Built success") + def content = readFile('dependencies.txt') + //log.info("reading content of file") + def lines = content.readLines() + //log.info("searching for Building lines") + def buildingLines = lines.findIndexValues { it.contains("Building") } + buildingLines << lines.size() + //log.info("Lines found ${buildingLines}") + + def checkBuildVersions = { startIndex, endIndex, appName -> + + //log.info("Function for check versions") + log.info("Checking ${appName} :") + if (startIndex < 0 || endIndex > lines.size() || startIndex > endIndex) { + error "Invalid range: startIndex = ${startIndex}, endIndex = ${endIndex}" + } + def relevantLines = lines[startIndex..(endIndex - 1)] + + //echo "Building lines found at indices: ${buildingLines}" + def mqLine = relevantLines.find { it.contains("mq-client-blocking") } + if (!mqLine) { + echo "mq-client-blocking not found in the module. Skipping further checks." + return + } + def mqVersionFull = mqLine.split(':')[-2]?.trim() + echo "mq-client-blocking version is: ${mqVersionFull}" + def mqVersion = sh(script: "echo \"${mqVersionFull}\" | awk -F 'RELEASE' '{print \$1}' | sed 's/\\.\$//'", returnStdout: true).trim() + echo "Processed MQ version is: ${mqVersion}" + + //def mqVersionFull = relevantLines.find { it.contains("mq-client-blocking") }?.split(':')[-2]?.trim() + //add code to remove the RELEASE keyword - DONE + //lexicographically or integer wise - interger wise + //echo "mq version is : ${mqVersionFull}" + //def mqVersion = sh(script: "echo \"${mqVersionFull}\" | awk -F 'RELEASE' '{print \$1}' | sed 's/\\.\$//'", returnStdout: true).trim() + + // def springKafkaVersion = relevantLines.find { it.contains("org.springframework.kafka") }?.split(':')[-2]?.trim() + // echo "SPF kafka ${springKafkaVersion}" + // def apacheKafkaVersion = relevantLines.find { it.contains("org.apache.kafka:kafka-clients") }?.split(':')[-2]?.trim() + // echo "apache kafka ${apacheKafkaVersion}" + + + def springKafkaLine = relevantLines.find { it.contains("org.springframework.kafka:spring-kafka:jar") } + def springKafkaVersion = null + if (springKafkaLine) { + springKafkaVersion = springKafkaLine.split(':')[-2]?.trim() + echo "springframework kafka version: ${springKafkaVersion}" + } else { + echo "org.springframework.kafka not found in the module" + } + + def apacheKafkaLine = relevantLines.find { it.contains("org.apache.kafka:kafka-clients:jar") } + def apacheKafkaVersion = null + if (apacheKafkaLine) { + apacheKafkaVersion = apacheKafkaLine.split(':')[-2]?.trim() + echo "Apache kafka version: ${apacheKafkaVersion}" + } else { + echo "org.apache.kafka:kafka-clients not found in the module." + } + + if (mqVersion) { + echo "mq-client-blocking version: ${mqVersionFull} ( ${mqVersion} )" + if (isVersionMQ(mqVersion, "3.0.18")) { + if (springKafkaVersion) { + //echo "org.springframework.kafka version: ${springKafkaVersion}" + if (!isVersionMQ(springKafkaVersion, "2.9.13")) { + echo "org.springframework.kafka version: ${springKafkaVersion}" + error "org.springframework.kafka version is lower than 2.9.13. Build failed." + } + } + if (apacheKafkaVersion) { + //echo "org.apache.kafka version: ${apacheKafkaVersion}" + if (!isVersionMQ(apacheKafkaVersion,"3.6.0")) { + echo "org.apache.kafka version: ${apacheKafkaVersion}" + error "org.apache.kafka version is lower than 3.6.0. Build failed." + } + } + echo "org.springframework.kafka version: ${springKafkaVersion} || org.apache.kafka version: ${apacheKafkaVersion}" + } + } else { + echo "mq-client-blocking is missing or has a version lower than 3.0.18. Skipping checks." + } + } + log.info("Starting version checks") + for (int i = 0; i < buildingLines.size() - 1; i++) { + //log.info("starting Loop") + def startIndex = (int) buildingLines[i] + def endIndex = (int) buildingLines[i + 1] + def appNameLine = lines[startIndex] + def appName = appNameLine.split("Building")[-1]?.trim() + checkBuildVersions(startIndex, endIndex, appName) + } + } + } + } + +} + catch ( Exception e) { + env.msg = "Error checking the MQ version. Please check console output for more details - ${e}" + env.error_msg_to_db = 'Error checking the MQ version' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +boolean isVersionMQ(String v1, String v2) { + def v1Parts = v1.tokenize('.') + def v2Parts = v2.tokenize('.') + + for (int i = 0; i < Math.max(v1Parts.size(), v2Parts.size()); i++) { + int v1Part = i < v1Parts.size() ? v1Parts[i].toInteger() : 0 + int v2Part = i < v2Parts.size() ? v2Parts[i].toInteger() : 0 + if (v1Part != v2Part) { + return v1Part > v2Part + } + } + return true +} + +/* +Function to clean up artifacts from GCS in case of failure +input arguments: +repo_name: repository name +branch_name: branch name +bucket_name: bucket name +*/ +def cleanupGCSArtifacts(String repo_name, String branch_name, String bucket_name) { + try { + log.info("########################## Cleaning up artifacts from GCS due to failure. ##########################") + def artifactPath = "gs://${bucket_name}/${repo_name}/${branch_name}/${env.TAG}/" + + // Check if artifacts exist before attempting cleanup + def artifactsExist = sh(returnStdout: true, script: "gsutil ls \"${artifactPath}\" 2>/dev/null || echo ''").trim() + + if (artifactsExist) { + log.info("Removing artifacts from: ${artifactPath}") + sh(script: "gsutil -m rm -r \"${artifactPath}\"") + log.info("Successfully cleaned up artifacts from GCS.") + } else { + log.info("No artifacts found to clean up.") + } + } catch (Exception cleanupException) { + log.warn("Warning: Failed to clean up artifacts from GCS - ${cleanupException.getMessage()}") + // Don't throw the cleanup exception to avoid masking the original failure + } +} + + + +def runCommandWithOutput(script_cmd) { + def exitCodeFile = ".maven_exit_code" + def outputFile = ".maven_build_output" + sh( + script: """#!/bin/bash + set +e + ${script_cmd} 2>&1 | tee ${outputFile} + echo \${PIPESTATUS[0]} > ${exitCodeFile} + exit 0 + """ + ) + + def exitCode = sh( + script: "cat ${exitCodeFile}", + returnStdout: true + ).trim() as int + + def output = sh( + script: "cat ${outputFile}", + returnStdout: true + ).trim() + + return [ + output: output, + status: exitCode + ] +} + + diff --git a/src/com/meesho/stages/buildNode.groovy b/src/com/meesho/stages/buildNode.groovy new file mode 100644 index 0000000..7d4c2c0 --- /dev/null +++ b/src/com/meesho/stages/buildNode.groovy @@ -0,0 +1,568 @@ +package com.meesho.stages + +import com.meesho.utilities.buTeamMapping +import com.meesho.utilities.constructTemplate +import com.meesho.utilities.getDockerParams +import com.meesho.utilities.addSSHKey +import com.meesho.utilities.dockerUtilities +import com.meesho.utilities.getYamlParameter + +def removePackageLock() { + sh(script: 'rm -rf package-lock.json') +} + +def buildNode(def build_cmd) { + try { + if (fileExists('package-lock.json')) { + sh(script: 'npm ci') + } + else { + sh(script: 'npm install') + } + sh(script: build_cmd) + } + catch ( Exception e ) { + env.msg = 'Error in building node packages . Please check console output for more details.' + env.error_msg_to_db = 'Error Building Node Packages' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +def getArtifactId(String repo_name) { + dir("$repo_name") { + if (fileExists('package.json')) { + return sh(returnStdout: true, script: 'jq -r .name package.json').trim() + } + } +} + +def getAwsSecret(def secret_name, def destination_file, def team, def bu) { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws secretsmanager get-secret-value --secret-id ${secret_name} --query SecretString --output text > ${destination_file}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + getVaultSecret("vault kv get -format=json meesho/${env.cicd_environment}/${bu}/${team}/${secret_name} | jq -r .data.data > ${destination_file}") + } +} + +def getNpmRc(def secret_name, def team, def bu) { + def npmrc_file = secret_name + '-npmrc-' + env.cicd_environment + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws secretsmanager get-secret-value --secret-id ${npmrc_file} --query SecretString --output text| jq -r .MEESHO_NPMRC_SECRET > .npmrc") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + getVaultSecret("vault kv get -format=json meesho/${env.cicd_environment}/${bu}/${team}/${npmrc_file} | jq -r .data.data.MEESHO_NPMRC_SECRET > .npmrc") + } +} + +def getPemFile(def secret_name, def team, def bu) { + def pem_secret_name = secret_name + '-secrets-' + env.cicd_environment + if (env.BUILD_ENV == 'stage') { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws secretsmanager get-secret-value --secret-id ${pem_secret_name} --query SecretString --output text| jq -r .public_secret_dev > 1_public_secret_dev.pem") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + getVaultSecret("vault kv get -format=json meesho/${env.cicd_environment}/${bu}/${team}/${pem_secret_name} | jq -r .data.data.public_secret_dev > 1_public_secret_dev.pem") + } + sh(script:"cat 1_public_secret_dev.pem | sed -e 's/-----BEGIN PUBLIC KEY-----/& \\n/' -e 's/-----END PUBLIC KEY-----/\\n-----END PUBLIC KEY-----/g' > public_secret_dev.pem") + } + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws secretsmanager get-secret-value --secret-id ${pem_secret_name} --query SecretString --output text| jq -r .public_secret_prod > 1_public_secret_prod.pem") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + getVaultSecret("vault kv get -format=json meesho/${env.cicd_environment}/${bu}/${team}/${pem_secret_name} | jq -r .data.data.public_secret_prod > 1_public_secret_prod.pem") + } + + sh(script:"cat 1_public_secret_prod.pem | sed -e 's/-----BEGIN PUBLIC KEY-----/& \\n/' -e 's/-----END PUBLIC KEY-----/\\n-----END PUBLIC KEY-----/g' > public_secret_prod.pem") +} + +def getEnvFile(def secret_name, def team, def bu, boolean useCacPath = false) { + def env_file = secret_name + '-env-' + env.cicd_environment + def gcp_env_file = secret_name + def destination_file = '.env' + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws secretsmanager get-secret-value --secret-id ${env_file} | jq --raw-output '.SecretString' | jq '.' | jq -r 'to_entries|map(\"\\(.key)=\\(.value|tostring)\")|.[]' > .env") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + def vault_path = useCacPath + ? "meesho/${env.cicd_environment}-cac/${bu}/${team}/${gcp_env_file}-client" + : "meesho/${env.cicd_environment}/${bu}/${team}/${gcp_env_file}" + getVaultSecret("vault kv get -format=json ${vault_path} | jq -r '.data.data | to_entries|map(\"\\(.key)=\\(.value|tostring)\")|.[]' > .env") + } +} + +/** + * Read appConfigEnabled from deployment yaml(s) in repo (deployments/<name>.yaml). + * Uses config.deployment_order for deployment names, or repo_name if not set. + * Aligns with deploy/Helm which reads appConfigEnabled from the same files. + */ +def isAppConfigEnabledFromDeployments(String repo_name, Map config) { + def deploymentNames = (config.deployment_order instanceof List && !config.deployment_order.isEmpty()) + ? config.deployment_order + : [repo_name] + def yamlObj = new getYamlParameter() + for (def deployment in deploymentNames) { + try { + def depYaml = yamlObj.getParam(repo_name, "deployments/${deployment}.yaml") + def enabled = depYaml?.appConfigEnabled + if (enabled == true || enabled?.toString()?.equalsIgnoreCase('true')) { + return true + } + } catch (Exception e) { + log.debug("No appConfigEnabled in ${repo_name}/deployments/${deployment}.yaml or file missing: ${e.message}") + continue + } + } + return false +} + +def getManifestJson(def secret_name, def team, def bu) { + def manifest_file = secret_name + '-manifest-' + env.cicd_environment + def destination_file = 'public/manifest.json' + getAwsSecret(manifest_file, destination_file, team, bu) +} + +def generateExcludePattern(String excludeS3Files) { + def files = excludeS3Files.split(/\s*,\s*/) // Split the string by comma and trim whitespace + return '|' + files.join('|') // Join file names with '|' as an "or" operator in regex +} + +def shouldSkipPm2Metrics(Map config) { + return config.skip_pm2_metrics?.toString()?.toBoolean() ?: false +} + +/* +Fuction to build maven docker repo +*/ + +def getCommitSHA(String repo_name) { + dir(repo_name) { + return sh(returnStdout: true, script: 'git log -1 --format=%H').trim() + } +} + +def buildDckr(Map config) { + // // Block all Node.js builds - throwing exception + // throw new Exception("Node.js builds are currently blocked, As a precautionary measure. Due to some packages that got hacked.") + def btObj = new buTeamMapping() + def constructObj = new constructTemplate() + def dparam_obj = new getDockerParams() + def addSSHKey = new addSSHKey() + def dockerUtilObj = new dockerUtilities() + + def team = btObj.get_team_initials(config.team) + def bu = btObj.get_bu_initials(config.bu) + def repo_name = config.repo_name + // CAC: read appConfigEnabled from deployment folder (deployments/<name>.yaml), same source as deploy/Helm + def useCacPath = isAppConfigEnabledFromDeployments(repo_name, config) + def build_cmd = config.build_cmd ?: 'npm run build' + // Resolve secret_name from deployment YAML app_name. + // For multi-deployment repos, env/secret files are kept in sync so the first valid app_name is used. + def secret_name = repo_name + def keep_package_lock = config.keep_package_lock != null ? config.keep_package_lock : true + def skip_npmrc = config.skip_npmrc != null ? config.skip_npmrc : true + if (config.secret_name) { + secret_name = config.secret_name + } else { + try { + def yamlObj = new getYamlParameter() + def deploymentsPath = "${repo_name}/deployments" + def yamlFilesOutput = sh( + script: "ls ${deploymentsPath}/*.yaml 2>/dev/null | xargs -r -n1 basename", + returnStdout: true + ).trim() + if (yamlFilesOutput) { + def yamlFiles = yamlFilesOutput.split('\n').collect { it.trim() }.findAll { it } + for (yamlFile in yamlFiles) { + def appConfig = yamlObj.getParam(deploymentsPath, yamlFile) + def appName = appConfig?.app_name?.toString()?.trim() + if (appName) { + echo "Resolved secret_name to ${appName} from ${deploymentsPath}/${yamlFile}" + secret_name = appName + break + } + } + } + if (secret_name == repo_name) { + echo "No app_name found in deployment yamls; using repo_name: ${repo_name}" + } + } catch (Exception e) { + echo "Error reading deployment yaml: ${e.getMessage()}; using repo_name: ${repo_name}" + } + } + def require_mainfest = config.require_mainfest ?: false + def pbac_enabled = config.pbac_enabled ?: false + def pbac_scope_name = config.pbac_scope_name ?: '' + def npm_install_arg = config.npm_install_arg ?: '' + if (!npm_install_arg) { + if (fileExists("${repo_name}/pnpm-lock.yaml") || build_cmd.contains('pnpm')) { + npm_install_arg = 'npm install -g pnpm@10.33.0 && pnpm install --frozen-lockfile' + } + else if (!keep_package_lock || !fileExists("${repo_name}/package-lock.json")) { + npm_install_arg = 'npm install' + } + // else: package-lock.json exists + keep_package_lock=true → Dockerfile uses `npm ci` + } + def require_pemfiles = config.require_pemfiles ?: false + def deployArgo = config.deployArgo ?: false + def artifactId = getArtifactId("${repo_name}") + // def region = dparam_obj.getRegion(env.cicd_environment) + // def registry = dparam_obj.getRegistry(env.cicd_environment) + def docker_repo = "${env.cicd_environment}/${team}/${repo_name.toLowerCase()}" + def push_to_s3 = config.push_to_s3 ?: false + def s3_path = config.s3_path ?: "meesho-${env.BUILD_ENV}-artifacts/${repo_name}/${env.BRANCH_NAME}" + if (env.INFRA_ENV == 'toolchain' && env.TOOLCHAIN_ENV) { + docker_repo = "${env.cicd_environment}/${env.TOOLCHAIN_ENV}/${team}/${repo_name.toLowerCase()}" + log.info("Toolchain: Overriding docker_repo to ${docker_repo}") + echo "docker_repo: ${docker_repo}" + + if (config.s3_path && config.push_to_s3) { + echo" {env.cicd_environment} should be stg due ot ovveride" + s3_path = s3_path.replaceFirst("/${env.cicd_environment}/", "/${env.TOOLCHAIN_ENV}/") + log.info("TOOLCHAIN OVERRIDE:") + log.info(" Original S3 Path: ${config.s3_path}") + log.info(" New Toolchain S3 Path: ${s3_path}") + log.info(" Toolchain Env ID: ${env.TOOLCHAIN_ENV}") + + } else if (!config.push_to_s3) { + log.info("Toolchain: push_to_s3 is false, skipping s3_path override as it wont be pushed") + } else if (!config.s3_path && config.push_to_s3) { + log.info("Toolchain: s3_path is not set but push_to_s3 is true, skipping s3_path override as it wont be pushed") + error("CRITICAL: 's3_path' is missing in config.yaml for the ${env.cicd_environment} environment. Toolchain builds require an s3_path.") + } + } + if (env.TOOLCHAIN_ENV) { + log.info("TOOLCHAIN_ENV: ${env.TOOLCHAIN_ENV}") + }else{ + log.info("TOOLCHAIN_ENV: not set") + } + def local_path = config.local_path ?: 'build/' + def acl = config.acl ? ' --acl ' + config.acl : '' + def custom_pm2_metrics = config.custom_pm2_metrics ?: false + def skip_pm2_metrics = shouldSkipPm2Metrics(config) + def include_s3_files = '' + def exclude_s3_files = '' + if (!s3_path.endsWith('/')) { + s3_path += '/' + } + if (!local_path.endsWith('/')) { + local_path += '/' + } + if (env.CLOUD_PROVIDER == 'AWS') { + if (config.include_s3_files) { + include_s3_files = ' --exclude "*"' + def include_file_map = config.include_s3_files.split(',') + for (pattern in include_file_map) { + include_s3_files += ' --include "'+pattern+'"' + } + } + exclude_s3_files = config.exclude_s3_files ? ' --exclude "'+config.exclude_s3_files+'"' : '' + } + else if (env.CLOUD_PROVIDER == 'GCP') { + if (config.include_s3_files) { + def includeFileMap = config.include_s3_files.split(',') + def file_types = '' + def i = 0 + includeFileMap.each { pattern -> + if ( i == 0 ) { + file_types = "${pattern.replaceAll('\\*.', '')}" + } + else { + file_types += "|${pattern.replaceAll('\\*.', '')}" + } + i++ + } + if ( file_types == '*' ) { + include_s3_files = '^(?!.*\\.*$)' + } + else { + include_s3_files = "^(?!.*\\.(${file_types})\$)" + } + echo "include_s3_files is ${include_s3_files}" + } + else { + include_s3_files = '^(?!.*\\.*$)' + } + + exclude_s3_files = config.exclude_s3_files != null && config.exclude_s3_files.trim()? generateExcludePattern(config.exclude_s3_files.trim()) : '' + + } + def phantomjs = config.phantomjs ?: false + def version = config.dockerBuildVersion.split('-')[-1] + def tag = dparam_obj.getTag(repo_name) + // TODO: create value binding for dockerfile render + + def skip_sonar = config.skip_sonar ?: false + if (env.hot_fix || env.INFRA_ENV == 'toolchain') { + skip_sonar = true + } + def testCMD = config.testCMD ?: 'test-report' + def scmType = 'branch' + def commit_sha = getCommitSHA(config.repo_name) + def docker_bindings = [ + 'buildRegistry': env.buildRegistry, + 'version': version, + 'build_cmd': build_cmd, + 'push_to_s3': push_to_s3, + 's3_path': s3_path, + 'local_path': local_path, + 'include_s3_files': include_s3_files, + 'exclude_s3_files': exclude_s3_files, + 'acl': acl, + 'phantomjs': phantomjs, + 'skip_sonar': skip_sonar, + 'testCMD': testCMD, + 'CLOUD_PROVIDER': env.CLOUD_PROVIDER, + 'npm_install_arg': npm_install_arg + ] + docker_bindings['arch'] = config.arch + docker_bindings['pbac_enabled'] = pbac_enabled + docker_bindings['pbac_scope_name'] = pbac_scope_name ?: '' + // Map BUILD_ENV to pbac format (stg/int/prd) + def pbac_env = env.cicd_environment + docker_bindings['pbac_env'] = pbac_env + docker_bindings['useCacPath'] = useCacPath + + // Validate pbac configuration + if (pbac_enabled && !pbac_scope_name) { + error('pbac_scope_name is required when pbac_enabled is true in config.yaml') + } + + stage(stageName('Creating build files')) { + dir(repo_name) { + if (!keep_package_lock) { + removePackageLock() + } + if (!skip_npmrc) { + getNpmRc(secret_name, team, bu) + } + + getEnvFile(secret_name, team, bu, useCacPath) + + if (require_pemfiles) { + getPemFile(secret_name, team, bu) + } + if (require_mainfest) { + getManifestJson(secret_name, team, bu) + } + def npm_registry = '' + if (fileExists('.npmrc')) { + npm_registry = sh( + returnStdout: true, + script: "grep '^@meesho:registry=' .npmrc | head -1 | cut -d'=' -f2 | tr -d '\\r\\n'" + ).trim() + echo "Detected npm registry: ${npm_registry ?: 'default (none found)'}" + } else { + echo ".npmrc not found, skipping registry extraction" + } + docker_bindings['npm_registry'] = npm_registry + } + } + + if (skip_sonar){ + stage(stageName('Checking quality gate')){ + log.info("Skipping the quality gate check as sonar scan is skipped or this is a hotfix ") + } + } + stage(stageName('Building docker images')) { + // Login to docker + try { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws ecr get-login-password --region ${env.region} | docker login --username AWS --password-stdin ${env.registry}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script:'gcloud auth configure-docker asia-southeast1-docker.pkg.dev --quiet') + } + } + catch (Exception e) { + env.msg = 'Error in Docker login' + env.error_msg_to_db = 'Error in Docker login' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + try { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws ecr describe-repositories --region ${env.region} --repository-names ${docker_repo} || aws ecr create-repository --region ${env.region} --repository-name ${docker_repo}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + echo 'Skipping - Registry Creation in GCP.' + } + } + catch (Exception e) { + env.msg = "Error in creating ECR repository ${docker_repo}" + env.error_msg_to_db = "Error in creating ECR repository ${docker_repo}" + currentBuild.result = 'FAILURE' + } + try { + dir(repo_name) { + addSSHKey.create() + withCredentials([usernamePassword(credentialsId: "svc-devops-meesho-token", usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) { + sh """ + echo "GITHUB_TOKEN=${GIT_TOKEN}" >> .env + echo "GIT_COMMIT_SHA='${commit_sha}'" >> .env + """ + } + withCredentials([string(credentialsId: env.sonarToken, variable: 'TOKEN')]) { + sh """ + echo "SONAR_HOST_URL='${env.sonarURL}'" >> .env + echo "SONAR_TOKEN='${TOKEN}'" >> .env + echo "SONAR_WS_TIMEOUT=120" >> .env + """ + if (env.CHANGE_ID) { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh "git fetch origin ${env.CHANGE_TARGET}:refs/remotes/origin/${env.CHANGE_TARGET}" + } + sh """ + echo "SONAR_CHANGE_ID='${env.CHANGE_ID}'" >> .env + echo "SONAR_CHANGE_BRANCH='${env.CHANGE_BRANCH}'" >> .env + echo "SONAR_CHANGE_TARGET='${env.CHANGE_TARGET}'" >> .env + """ + scmType = 'pr' + } + else { + sh """ + echo "SONAR_BRANCH_NAME='${env.BRANCH_NAME}'" >> .env + """ + } + echo 'Make sure to update script section in package.json for Sonar Analysis to be successfull.' + docker_bindings['scmType'] = scmType + } + if (!fileExists('Dockerfile')) { + constructObj.renderTemplate(docker_bindings, 'node-Dockerfile', 'Dockerfile-' + artifactId) + sh "cat Dockerfile-${artifactId}" + if (push_to_s3) { + if (env.CLOUD_PROVIDER == 'AWS') { + sh """ + echo 'FROM amazon/aws-cli:2.2.0 as push_env' >> Dockerfile-${artifactId} + echo 'COPY --from=build-env /usr/src/app /app' >> Dockerfile-${artifactId} + echo 'RUN ls -al && aws s3 cp /app/${local_path} s3://${s3_path}${include_s3_files}${exclude_s3_files} --recursive --cache-control max-age=31536000,public${acl}' >> Dockerfile-${artifactId} + """ + } + else if (env.CLOUD_PROVIDER == 'GCP' ) { + sh """ + echo 'FROM asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/google-cloud-sdk:458.0.0-alpine as push_env' >> Dockerfile-${artifactId} + echo 'COPY --from=build-env /usr/src/app /app' >> Dockerfile-${artifactId} + # echo 'RUN ls -al /app && find /app/${local_path} -type f \\( -name "*.js" -o -name "*.map" \\) && find /app/${local_path} -type f \\( -name "*.js" -o -name "*.map" \\) | wc -l && gsutil -m cp -r /app/${local_path}* gs://${s3_path}' >> Dockerfile-${artifactId} + echo "RUN ls -al /app && gsutil -m rsync -r -x \'${include_s3_files} ${exclude_s3_files}\' /app/${local_path} gs://${s3_path}" >> Dockerfile-${artifactId} + echo "RUN if [ -f /app/${local_path}index.html ]; then gsutil cp /app/${local_path}index.html gs://${s3_path}; fi" >> Dockerfile-${artifactId} + """ + } + } + + if (phantomjs) { + sh """ + echo 'FROM build-env' >> Dockerfile-${artifactId} + echo 'RUN apt-get update && apt-get install -y libfontconfig' >> Dockerfile-${artifactId} + echo 'WORKDIR /usr/src/app' >> Dockerfile-${artifactId} + echo 'RUN npm install pm2 -g' >> Dockerfile-${artifactId} + echo 'RUN pm2 install pm2-metrics' >> Dockerfile-${artifactId} + if [ "${custom_pm2_metrics}" = "true" ]; then + echo 'RUN pm2 install pm2-prom-module' >> Dockerfile-${artifactId} + echo 'RUN pm2 set pm2-prom-module:port 9200' >> Dockerfile-${artifactId} + echo 'RUN pm2 restart pm2-prom-module' >> Dockerfile-${artifactId} + fi + echo 'RUN rm -rf /root/.ssh/id_rsa && apt-get remove -y git openssh-client bzip2' >> Dockerfile-${artifactId} + """ + } + else { + sh """ + echo 'FROM ${env.buildRegistry}/build/node:${version}-alpine-secure-multiarch_v1.0' >> Dockerfile-${artifactId} + echo 'RUN npm install pm2 -g' >> Dockerfile-${artifactId} + if [ "${skip_pm2_metrics}" != "true" ]; then + echo 'RUN pm2 install pm2-metrics' >> Dockerfile-${artifactId} + fi + if [ "${custom_pm2_metrics}" = "true" ]; then + echo 'RUN pm2 install pm2-prom-module' >> Dockerfile-${artifactId} + echo 'RUN pm2 set pm2-prom-module:port 9200' >> Dockerfile-${artifactId} + echo 'RUN pm2 restart pm2-prom-module' >> Dockerfile-${artifactId} + fi + echo 'WORKDIR /app' >> Dockerfile-${artifactId} + echo 'COPY --from=build-env /usr/src/app /app' >> Dockerfile-${artifactId} + """ + } + if (docker_bindings['useCacPath']) { + sh "echo 'RUN truncate -s 0 .env' >> Dockerfile-${artifactId}" + } + sh "cat Dockerfile-${artifactId}" + def imageList = "${env.buildRegistry}/build/node:${version}-alpine-secure-multiarch_v1.0 " + + "asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin/devops/google-cloud-sdk:458.0.0-alpine " + + "${env.buildRegistry}/build/node:${version}-slim-secure-multiarch_v2.0" + + sh """ + max_attempts=5 + images=\"$imageList\" + + for image in \$images; do + attempt=1 + until docker pull \"\$image\"; do + if [ \$attempt -eq \$max_attempts ]; then + echo \"Failed to pull \$image after \$attempt attempts.\" + exit 1 + fi + echo \"Pull failed for \$image, retrying in 2 seconds... (Attempt \$attempt/\$max_attempts)\" + attempt=\$((attempt + 1)) + sleep 2 + done + done + """ + if (env.cicd_environment != 'ftr' || env.INFRA_ENV == 'toolchain') { + def docker_cmd = "export DOCKER_BUILDKIT=0;docker build --tag ${env.registry}/${docker_repo}:${tag} -f Dockerfile-${artifactId} . " + sh(script: docker_cmd) + dockerUtilObj.retryDockerPush("docker push ${env.registry}/${docker_repo}:${tag}") + } + else { + def docker_cmd = "export DOCKER_BUILDKIT=0;docker build --tag ${env.registry}/${docker_repo}:${tag} -f Dockerfile-${artifactId} ." + sh(script: docker_cmd) + } + //Remove dockerfile + sh(script: "rm -rf Dockerfile-${artifactId}") + } + else { + if (env.cicd_environment != 'ftr' || env.INFRA_ENV == 'toolchain') { + sh(script: "export DOCKER_BUILDKIT=0;docker build --tag ${env.registry}/${docker_repo}:${tag} . ") + dockerUtilObj.retryDockerPush("docker push ${env.registry}/${docker_repo}:${tag}") + } + else { + log.info("Skipping Docker Push - ${env.cicd_environment} env") + sh(script: "export DOCKER_BUILDKIT=0;docker build --tag ${env.registry}/${docker_repo}:${tag} .") + } + } + } + } + catch (Exception e) { + env.msg = 'Error in building DockerFile Or Pushing To ECR' + env.error_msg_to_db = env.msg + log.error(env.msg + '\n' + e.toString()) + currentBuild.result = env.FAILURE + throw e + } + return [tag , deployArgo] + } +} + + +def getVaultSecret(String vault_cmd) { + log.info("Fetching secrets from Vault with vault-${env.cicd_environment}-token with CMD - ${vault_cmd}") + if (env.INFRA_ENV == 'toolchain' && env.TOOLCHAIN_ENV) { + boolean skipToolchainOverride = vault_cmd.contains("npmrc") || vault_cmd.contains("-secrets-") + if (!skipToolchainOverride) { + vault_cmd = vault_cmd.replace("meesho/${env.cicd_environment}-cac/", "meesho/toolchain/${env.TOOLCHAIN_ENV}/${env.cicd_environment}-cac/") + vault_cmd = vault_cmd.replace("meesho/${env.cicd_environment}/", "meesho/toolchain/${env.TOOLCHAIN_ENV}/${env.cicd_environment}/") + log.info("TOOLCHAIN OVERRIDE (Application Config):") + log.info(" New Toolchain Vault CMD: ${vault_cmd}") + } else { + log.info("TOOLCHAIN: Reading from standard (non-toolchain) Vault path for npmrc / pem secret.") + } + } + + withCredentials([string(credentialsId: "${env.vaultToken}", variable: 'TOKEN')]) { + env.VAULT_ADDR = "${env.vaultURL}" + env.VAULT_TOKEN = "${TOKEN}" + sh(script:"${vault_cmd}") + env.VAULT_TOKEN = 'empty' + env.VAULT_ADDR = env.VAULT_TOKEN + } +} diff --git a/src/com/meesho/stages/buildObjHelper.groovy b/src/com/meesho/stages/buildObjHelper.groovy new file mode 100644 index 0000000..3aaf055 --- /dev/null +++ b/src/com/meesho/stages/buildObjHelper.groovy @@ -0,0 +1,42 @@ +package com.meesho.stages + +def run(String build_tool){ + try { + switch (build_tool) { + case 'maven': + return new buildMaven() + break; + case 'docker': + return new buildDocker() + break; + case ~/^maven-.*/: + return new buildMaven() + break; + case ~/^python-.*/: + return new buildPython() + break; + case ~/^node-.*/: + return new buildNode() + break; + case ~/^rust.*/: + return new buildRust() + break; + case ~/^go.*/: + return new buildGo() + break; + case 'gradle': + return new buildGradle() + break; + case 'php': + return new buildPhp() + default: + return defaultBuild() + } + } + catch (Exception e) { + env.msg = 'Error in selecting the build tool (check the spell) Error: ' + e.toString() + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} diff --git a/src/com/meesho/stages/buildPhp.groovy b/src/com/meesho/stages/buildPhp.groovy new file mode 100644 index 0000000..59d4e6c --- /dev/null +++ b/src/com/meesho/stages/buildPhp.groovy @@ -0,0 +1,104 @@ +package com.meesho.stages + +import com.meesho.utilities.buTeamMapping +import com.meesho.utilities.constructTemplate +import com.meesho.utilities.getDockerParams + +def buildDckr(Map config){ + def btObj = new buTeamMapping() + def constructObj = new constructTemplate() + def dparam_obj = new getDockerParams() + + def team = btObj.get_team_initials(config.team) + def repo_name = config.repo_name + def deployArgo = config.deployArgo ?: false + // def region = dparam_obj.getRegion(env.cicd_environment) + // def registry = dparam_obj.getRegistry(env.cicd_environment) + def docker_repo = "${env.cicd_environment}/${team}/${repo_name.toLowerCase()}" + def tag = dparam_obj.getTag(repo_name) + def docker_bindings = [ + "repo_name": repo_name, + 'buildRegistry': env.buildRegistry + + ] + if (config.containsKey("copy_file")){ + if (env.CLOUD_PROVIDER == 'AWS'){ + def recursive = config.copy_file.recursive ? " --recursive" : "" + dir(repo_name){ + dir('copied_files'){ + sh(script:"aws s3 cp${recursive} ${config.copy_file.path} .") + } + } + } else if (env.CLOUD_PROVIDER == 'GCP'){ + def recursive = config.copy_file.recursive ? " -r" : "" + dir(repo_name){ + dir('copied_files'){ + sh(script:"gsutil cp${recursive} ${config.copy_file.path} .") + } + } + } + docker_bindings["copy_file"] = true + docker_bindings["copy_target"] = config.copy_file.target ?: "/opt/target" + } + else{ + docker_bindings["copy_file"] = false + } + stage(stageName('Building docker images')){ + // Login to docker + try { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws ecr get-login-password --region ${env.region} | docker login --username AWS --password-stdin ${env.registry}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script:'gcloud auth configure-docker asia-southeast1-docker.pkg.dev --quiet') + } + } + catch(Exception e) { + env.msg = "Error in Docker login" + env.error_msg_to_db = "Error in Docker login" + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + try{ + if (env.CLOUD_PROVIDER == "AWS"){ + sh(script:"aws ecr describe-repositories --region ${env.region} --repository-names ${docker_repo} || aws ecr create-repository --region ${env.region} --repository-name ${docker_repo}") + } + else if (env.CLOUD_PROVIDER == 'GCP'){ + echo "Skipping - Registry Creation in GCP." + } + } + catch(Exception e){ + env.msg = "Error in creating ECR repository ${docker_repo}" + env.error_msg_to_db = "Error in creating ECR repository ${docker_repo}" + currentBuild.result = "FAILURE" + } + try { + dir(repo_name){ + if(!fileExists("Dockerfile")){ + constructObj.renderTemplate(docker_bindings,'php-Dockerfile','Dockerfile-php') + if (env.cicd_environment != 'ftr') { + sh(script: "docker build --tag ${env.registry}/${docker_repo}:${tag} -f Dockerfile-php . && docker push ${env.registry}/${docker_repo}:${tag}") + } + else { + log.info("Skipping Docker builds for PHP in ${env.cicd_environment} env") + } + + //Remove dockerfile + sh(script: "rm -rf Dockerfile-php") + } + else { + sh(script: "docker build --tag ${env.registry}/${docker_repo}:${tag} . && docker push ${env.registry}/${docker_repo}:${tag}") + } + } + } + catch (Exception e) { + env.msg = "Error in building DockerFile Or Pushing To artifact registry" + env.error_msg_to_db = "Error in building DockerFile Or Pushing To image registry" + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + } + return [tag , deployArgo] +} diff --git a/src/com/meesho/stages/buildPython.groovy b/src/com/meesho/stages/buildPython.groovy new file mode 100644 index 0000000..45029c3 --- /dev/null +++ b/src/com/meesho/stages/buildPython.groovy @@ -0,0 +1,221 @@ +package com.meesho.stages + +import com.meesho.utilities.buTeamMapping +import com.meesho.utilities.constructTemplate +import com.meesho.utilities.getDockerParams +import com.meesho.utilities.addSSHKey +import com.meesho.utilities.dockerUtilities +/* +Function to get the version from pom.xml +input arguments: +repo_name: String parameter to change the directory where pom.xml is located +*/ +// def getArtifactId(String repo_name){ +// dir("$repo_name"){ +// if (fileExists("package.json")) { +// return sh(returnStdout: true, script: 'jq -r .name package.json').trim() +// } +// } +// } + +// def getAwsSecret(def secret_name, def destination_file){ +// sh (script:"aws secretsmanager get-secret-value --secret-id ${secret_name} --query SecretString --output text > ${destination_file}") +// } + +// def getCommitid(String repo_name){ +// dir("$repo_name"){ +// return sh(returnStdout: true, script: 'git log -1 --format=%h').trim() +// } +// } + +// def getNpmRc(def secret_name){ +// def npmrc_file = secret_name + "-npmrc" +// sh (script:"aws secretsmanager get-secret-value --secret-id ${npmrc_file} --query SecretString --output text| jq -r .MEESHO_NPMRC_SECRET > .npmrc") +// } + +// def getPemFile(def secret_name){ +// def pem_secret_name = secret_name + "-secrets" +// if(env.BUILD_ENV == 'stage'){ +// sh (script:"aws secretsmanager get-secret-value --secret-id ${pem_secret_name} --query SecretString --output text| jq -r .public_secret_dev > 1_public_secret_dev.pem") +// sh (script:"cat 1_public_secret_dev.pem | sed -e 's/-----BEGIN PUBLIC KEY-----/& \\n/' -e 's/-----END PUBLIC KEY-----/\\n-----END PUBLIC KEY-----/g' > public_secret_dev.pem") +// } +// sh (script:"aws secretsmanager get-secret-value --secret-id ${pem_secret_name} --query SecretString --output text| jq -r .public_secret_prod > 1_public_secret_prod.pem") +// sh (script:"cat 1_public_secret_prod.pem | sed -e 's/-----BEGIN PUBLIC KEY-----/& \\n/' -e 's/-----END PUBLIC KEY-----/\\n-----END PUBLIC KEY-----/g' > public_secret_prod.pem") +// } + +// def getEnvFile(def secret_name){ +// def env_file = secret_name + "-env" +// def destination_file = ".env" +// sh(script:"aws secretsmanager get-secret-value --secret-id ${env_file} | jq --raw-output '.SecretString' | jq '.' | jq -r 'to_entries|map(\"\\(.key)=\\(.value|tostring)\")|.[]' > .env") +// } + +// def getManifestJson(def secret_name){ +// def manifest_file = secret_name+"-manifest" +// def destination_file = "public/manifest.json" +// getAwsSecret(manifest_file,destination_file) +// } +/* +Fuction to build maven docker repo +*/ + +def buildDckr(Map config) { + def btObj = new buTeamMapping() + def constructObj = new constructTemplate() + def dparam_obj = new getDockerParams() + def addSSHKey = new addSSHKey() + + def team = btObj.get_team_initials(config.team) + def repo_name = config.repo_name + def deployArgo = config.deployArgo ?: false + def modules_requirements_file = config.modules_requirements_file ?: 'requirements.txt' + def docker_repo = "${env.cicd_environment}/${team}/${repo_name.toLowerCase()}" + def tag = dparam_obj.getTag(repo_name) + if (env.INFRA_ENV == 'toolchain') { + def dockerUtilObj = new dockerUtilities() + if (dockerUtilObj.imageExists(env.registry, docker_repo, tag)) { + log.info("Toolchain: Image found in registry for ${docker_repo}:${tag}. Skipping build step.") + return [tag, deployArgo] + } else { + log.info("Toolchain: Image missing for ${docker_repo}:${tag}. Proceeding with build.") + } + } + // TODO: create value binding for dockerfile render + def docker_bindings = [ + 'buildRegistry': env.buildRegistry, + 'modules_requirements_file': modules_requirements_file, + 'arch': config.arch, + 'buildRegistry': env.buildRegistry + ] + docker_bindings['arch'] = config.arch + // stage("Create build files"){ + // dir(repo_name){ + // getNpmRc(secret_name) + // getEnvFile(secret_name) + // if (require_pemfiles){getPemFile(secret_name)} + // if (require_mainfest){getManifestJson(secret_name)} + // } + // } + stage(stageName('Building docker images')) { + // Login to docker + try { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws ecr get-login-password --region ${env.region} | docker login --username AWS --password-stdin ${env.registry}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script:'gcloud auth configure-docker asia-southeast1-docker.pkg.dev --quiet') + } + } + catch (Exception e) { + env.msg = 'Error in Docker login' + env.error_msg_to_db = 'Error in Docker login' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + if (config.containsKey('copy_file')) { + def recursive = config.copy_file.recursive ? ' --recursive' : '' + docker_bindings['copy_file_path'] = config.copy_file.path + dir(repo_name) { + dir('copied_files') { + if (env.CLOUD_PROVIDER == 'AWS') { + // delete any old data + sh(script:'rm -rf *') + sh(script:"aws s3 cp${recursive} ${config.copy_file.path} .") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + // delete any old data + docker_bindings['recursive'] = '-r' + sh(script:'rm -rf *') + sh(script:"gsutil cp ${docker_bindings.recursive} ${docker_bindings.copy_file_path} .") + } + } + } + docker_bindings['copy_file'] = true + docker_bindings['copy_target'] = config.copy_file.target ?: '/app/' + } + else { + docker_bindings['copy_file'] = false + } + try { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws ecr describe-repositories --region ${env.region} --repository-names ${docker_repo} || aws ecr create-repository --region ${env.region} --repository-name ${docker_repo}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + echo 'Skipping - Registry Creation in GCP.' + } + } + catch (Exception e) { + env.msg = "Error in creating ECR repository ${docker_repo}" + env.error_msg_to_db = "Error in creating ECR repository ${docker_repo}" + currentBuild.result = env.FAILURE + } + try { + dir(repo_name) { + addSSHKey.create() + if (!fileExists('Dockerfile')) { + // print(config) + constructObj.renderTemplate(docker_bindings,config.dockerBuildVersion+'-Dockerfile','Dockerfile-'+repo_name) + sh "cat Dockerfile-${repo_name}" + // withCredentials([string(credentialsId: 'meesho-github-ssh-prv-key', variable: 'SSH_PRIVATE_KEY_S')]) { + // withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'meesho-ssh-github-key', \ + // keyFileVariable: 'SSH_PRIVATE_KEY', \ + // passphraseVariable: '', \ + // usernameVariable: '')]) { + //withCredentials([string(credentialsId: 'git_private_key', variable: 'gitkey')]) { + // sh """ + // set +x + // docker build --tag ${env.registry}/${docker_repo}:${tag} --build-arg SSH_PRIVATE_KEY="\$(cat ~/.ssh/id_github_jenkins)" -f "Dockerfile-${repo_name}" . && docker push ${env.registry}/${docker_repo}:${tag} + // set -x + // """ + if (env.CLOUD_PROVIDER == 'AWS') { + sh """ + set +x + docker build --tag ${env.registry}/${docker_repo}:${tag} --build-arg SSH_PRIVATE_KEY="\$(cat ~/.ssh/id_github_jenkins)" -f "Dockerfile-${repo_name}" . + docker push ${env.registry}/${docker_repo}:${tag} + set -x + """ + } + else if (env.CLOUD_PROVIDER == 'GCP') { + if (env.cicd_environment != 'ftr' || env.INFRA_ENV == 'toolchain') { + sh """ + set +x + docker build --tag ${env.registry}/${docker_repo}:${tag} --build-arg SSH_PRIVATE_KEY="\$(cat ~/.ssh/id_github_jenkins)" -f "Dockerfile-${repo_name}" . + docker push ${env.registry}/${docker_repo}:${tag} + set -x + """ + } + else { + log.info("Skipping Docker builds for Python in ${env.cicd_environment} env") + } + } + // } + // } + //Remove dockerfile + sh(script: "rm -rf Dockerfile-${repo_name}") + } + else { + sh "cat Dockerfile" + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: "docker buildx build --platform linux/arm64,linux/amd64 --tag ${env.registry}/${docker_repo}:${tag} --push .") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh """ + set +x + docker build --tag ${env.registry}/${docker_repo}:${tag} --build-arg SSH_PRIVATE_KEY="\$(cat ~/.ssh/id_github_jenkins)" -f "Dockerfile" . + docker push ${env.registry}/${docker_repo}:${tag} + set -x + """ + } + } + } + } + catch (Exception e) { + env.msg = "Error in building DockerFile Or Pushing To ECR. For Full Error Details - ${e}" + env.error_msg_to_db = 'Error in building DockerFile Or Pushing To ECR' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + return [tag , deployArgo] + } +} diff --git a/src/com/meesho/stages/buildRust.groovy b/src/com/meesho/stages/buildRust.groovy new file mode 100644 index 0000000..aeab669 --- /dev/null +++ b/src/com/meesho/stages/buildRust.groovy @@ -0,0 +1,315 @@ +package com.meesho.stages + +import com.meesho.utilities.buTeamMapping +import com.meesho.utilities.constructTemplate +import com.meesho.utilities.getDockerParams +import com.meesho.utilities.addSSHKey + + + +def buildDckr(Map config) { + env.RUSTPRIVATE = 'github.com/Meesho' + def btObj = new buTeamMapping() + def constructObj = new constructTemplate() + def dparam_obj = new getDockerParams() + def addSSHKey = new addSSHKey() + def deployArgo = config.deployArgo ?: true + + def team = btObj.get_team_initials(config.team) + def modules = config.modules ?: ['module_less'] + def repo_name = config.repo_name + def docker_repo = "${env.cicd_environment}/${team}/${repo_name.toLowerCase()}" + def tag = dparam_obj.getTag(repo_name) + def buildx = config.containsKey('buildx') ? config.buildx : true + def docker_bindings = [:] + def skip_sonar = config.skip_sonar ?: false + def version = config.dockerBuildVersion.split('-')[-1] + def repoType = config.repo_type ?: 'microservice' + docker_bindings['version'] = version + docker_bindings['base_dir'] = config.base_dir ?: false + docker_bindings['buildRegistry'] = env.buildRegistry + docker_bindings['build_packages'] = getSystemPackages(config, 'build_packages') + docker_bindings['runtime_packages'] = getSystemPackages(config, 'runtime_packages') + stage('Build docker images') { + + + // Login to docker + try { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws ecr get-login-password --region ${env.region} | docker login --username AWS --password-stdin ${env.registry}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + + sh(script:'gcloud auth configure-docker asia-southeast1-docker.pkg.dev --quiet') + } + } + catch (Exception e) { + env.msg = 'Error in Docker login' + env.error_msg_to_db = env.msg + log.error(env.msg + '. Error: ' + e.toString()) + currentBuild.result = env.FAILURE + throw e + } + if (config.containsKey('copy_file')) { + def recursive = config.copy_file.recursive ? ' --recursive' : '' + dir(repo_name) { + dir('copied_files') { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws s3 cp${recursive} ${config.copy_file.path} .") + } else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script:"gsutil cp${recursive} ${config.copy_file.path} .") + } + } + } + docker_bindings['copy_file'] = true + docker_bindings['copy_target'] = config.copy_file.target ?: '/app/' + docker_bindings['base_dir'] = config.base_dir ?: false + } + else { + docker_bindings['copy_file'] = false + } + + try { + dir(repo_name) { + sonar_scan(repo_name, skip_sonar, version ) + + if (repoType != 'microservice') { + log.info("Skipping Docker build for ${repoType} repo type") + return [tag, deployArgo] + } + + addSSHKey.create() + if (!fileExists('Dockerfile')) { + for (module in modules) { + def module_name = (module instanceof LinkedHashMap) ? module.keySet()[0] : module + docker_bindings['module_property'] = (module instanceof LinkedHashMap) ? module[module_name] : [ : ] + docker_bindings['module'] = module_name + docker_bindings['binary_name'] = (module == 'module_less') ? repo_name : module_name + constructObj.renderTemplate(docker_bindings, 'rust-Dockerfile', 'Dockerfile-' + module_name) + sh "cat Dockerfile-${module_name}" + module_repo = (module == 'module_less') ? docker_repo : docker_repo + '/' + module_name + try { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script:"aws ecr describe-repositories --region ${env.region} --repository-names ${module_repo} || aws ecr create-repository --region ${env.region} --repository-name ${module_repo}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + echo 'Skipping - Registry Creation in GCP.' + } + } + catch (Exception e) { + env.msg = "Error in creating ECR repository ${module_repo}" + env.error_msg_to_db = "Error in creating ECR repository ${module_repo}" + currentBuild.result = 'FAILURE' + } + if (buildx) { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: "set +x && docker buildx build --platform linux/arm64,linux/amd64 --tag ${env.registry}/${module_repo}:${tag} -f Dockerfile-${module_name} --push .") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + if (env.cicd_environment != 'ftr') { + sh(script: "set +x && docker build --tag ${env.registry}/${module_repo}:${tag} -f Dockerfile-${module_name} . && docker push ${env.registry}/${module_repo}:${tag}") + } + else { + log.info("Skipping Docker Push - ${env.cicd_environment} env") + sh(script: "set +x && docker build --tag ${env.registry}/${module_repo}:${tag} -f Dockerfile-${module_name} .") + } + } + } + else { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: "set +x && docker build --tag ${env.registry}/${module_repo}:${tag} -f Dockerfile-${module_name} . && docker push ${env.registry}/${module_repo}:${tag}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + if (env.cicd_environment != 'ftr') { + sh(script: """ + set +x + # Setup buildx for ARM64 builds + docker buildx rm mybuilder || true + docker buildx create --name mybuilder --driver docker-container --bootstrap + docker buildx use mybuilder + + # Build for ARM64 + docker buildx build --platform linux/arm64 --tag ${env.registry}/${module_repo}:${tag} -f Dockerfile-${module_name} --push . + """) + } + else { + log.info("Skipping Docker Push - ${env.cicd_environment} env") + sh(script: "set +x && docker build --tag ${env.registry}/${module_repo}:${tag} -f Dockerfile-${module_name} .") + } + } + } + //Remove dockerfile + sh(script: "rm -rf Dockerfile-${module_name}") + } + } + else { + sh "cat Dockerfile" + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: "docker buildx build --platform linux/arm64,linux/amd64 --tag ${env.registry}/${docker_repo}:${tag} --push .") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script: "docker buildx build --platform linux/arm64,linux/amd64 --tag ${env.registry}/${docker_repo}:${tag} --push .") + if (env.cicd_environment != 'ftr') { + sh(script: "docker buildx build --platform linux/arm64,linux/amd64 --tag ${env.registry}/${docker_repo}:${tag} --push .") + } + else { + log.info("Skipping Docker Push - ${env.cicd_environment} env") + sh(script: "docker buildx build --platform linux/arm64,linux/amd64 --tag ${env.registry}/${docker_repo}:${tag} .") + } + } + } + } + } + catch (Exception e) { + env.msg = 'Error in building DockerFile Or Pushing To ECR' + env.error_msg_to_db = env.msg + log.error(env.msg + '. Error: ' + e.toString()) + currentBuild.result = env.FAILURE + throw e + } + return [tag, deployArgo] + } +} + +// Services may need native libraries that are not part of a shared base image +// (for example, database client headers used by a crate's build script). The +// keys are language-neutral for future reuse; currently only the Rust builder +// consumes them. Restrict values to Debian package names so config data cannot +// alter the rendered Dockerfile instruction. +def getSystemPackages(Map config, String key) { + def configuredPackages = config[key] + if (configuredPackages == null) { + return [] + } + if (!(configuredPackages instanceof List)) { + throw new IllegalArgumentException("${key} must be a YAML list of system package names") + } + + def packages = configuredPackages.collect { packageName -> packageName?.toString()?.trim() } + if (packages.any { packageName -> !packageName || !(packageName ==~ /^[a-z0-9][a-z0-9+.-]*$/) }) { + throw new IllegalArgumentException("${key} contains an invalid system package name") + } + return packages.unique() +} + + +/* +Fuction to execute sonar scan + Input Arguments: + repo_name: repository name + skip_sonar: boolena parameter to skip sonar scan +*/ + +def sonar_scan(String repo_name, boolean skip_sonar , String version ) { + try { + stage('Run sonar scan') { + if (!skip_sonar) { + + withSonarQubeEnv('sonarqube-test') { + if (env.CHANGE_ID) { + + if (env.CLOUD_PROVIDER == 'AWS') { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh "git fetch origin ${env.CHANGE_TARGET}:refs/remotes/origin/${env.CHANGE_TARGET}" + } + sh(script: "mvn sonar:sonar -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.repository=Meesho/${repo_name} -Dsonar.pullrequest.key=${env.CHANGE_ID} -Dsonar.pullrequest.branch=${env.CHANGE_BRANCH} -Dsonar.pullrequest.base=${env.CHANGE_TARGET}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh "git fetch origin ${env.CHANGE_TARGET}:refs/remotes/origin/${env.CHANGE_TARGET}" + } + + sh(script: "curl -LO https://go.dev/dl/go${go_version}.linux-amd64.tar.gz") + sh(script: "tar -xvzf go${go_version}.linux-amd64.tar.gz -C /usr/local",returnStdout: true) + sh(script:"rm -Rf go${go_version}.linux-amd64.tar.gz") + // Set Go environment variables + env.PATH = "/usr/local/go/bin:${env.PATH}" + + // Run Go mod tidy and tests + sh (script: "go mod tidy") + int testExitCode = sh(script: "go test -short -coverprofile=./cov.out ./...", returnStatus: true) + if (testExitCode != 0) { + sh(script: "echo Go tests failed, but the pipeline will continue.") + } + + // Download and extract Sonar Scanner + sh(script: "curl -LO https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-5.0.1.3006-linux.zip") + sh(script: "unzip -o sonar-scanner-cli-5.0.1.3006-linux.zip -d /usr/local/sonar-scanner") + sh(script: "rm -Rf sonar-scanner-cli-5.0.1.3006-linux.zip") + env.PATH = "/usr/local/sonar-scanner/sonar-scanner-5.0.1.3006-linux/bin:${env.PATH}" + + // Run Sonar Scanner + sh(script:"sonar-scanner -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.repository=Meesho/${repo_name} -Dsonar.pullrequest.key=${env.CHANGE_ID} -Dsonar.pullrequest.branch=${env.CHANGE_BRANCH} -Dsonar.pullrequest.base=${env.CHANGE_TARGET} -Dsonar.go.coverage.reportPaths=./cov.out ") + } + } + else { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: "mvn sonar:sonar -Dsonar.branch.name=${env.BRANCH_NAME}") + } + else if (env.CLOUD_PROVIDER == 'GCP') { + + sh(script: "curl -LO https://go.dev/dl/go${go_version}.linux-amd64.tar.gz") + sh(script: "tar -xvzf go${go_version}.linux-amd64.tar.gz -C /usr/local",returnStdout: true) + sh(script:"rm -Rf go${go_version}.linux-amd64.tar.gz") + // Set Go environment variables + env.PATH = "/usr/local/go/bin:${env.PATH}" + + // Run Go mod tidy and tests + sh (script: "go mod tidy") + int testExitCode = sh(script: "go test -short -coverprofile=./cov.out ./...", returnStatus: true) + if (testExitCode != 0) { + sh(script: "echo Go tests failed, but the pipeline will continue.") + } + + // Download and extract Sonar Scanner + sh(script: "curl -LO https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-5.0.1.3006-linux.zip") + sh(script: "unzip -o sonar-scanner-cli-5.0.1.3006-linux.zip -d /usr/local/sonar-scanner") + sh(script: "rm -Rf sonar-scanner-cli-5.0.1.3006-linux.zip") + env.PATH = "/usr/local/sonar-scanner/sonar-scanner-5.0.1.3006-linux/bin:${env.PATH}" + + // Run Sonar Scanner + sh(script:"sonar-scanner -Dsonar.go.coverage.reportPaths=./cov.out -Dproject.settings=`pwd`/sonar-project.properties -Dsonar.branch.name=${env.BRANCH_NAME}") + + + } + } + } + + } + else { + log.info('Skipping - Sonar Scan') + } + } + stage("Quality Gate"){ + + if (skip_sonar){ + log.info("Sonar scan is skipped. Marking this stage as passed.") + } + else if (!env.CHANGE_ID){ + log.info("Skipping quality gate check on Branches. Marking this stage as passed.") + } + else{ + timeout(time: 600, unit: 'SECONDS') { + def qg = waitForQualityGate() + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE'){ + if (qg.status != 'OK') { + log.warn("Quality gate failed: ${qg.status}, but continuing pipeline execution.") + } + } + } + } + } + } + catch (Exception e) { + dir("$repo_name") { + withSonarQubeEnv('sonarqube-test') { + if (env.CLOUD_PROVIDER == 'AWS') { + sh(script: 'mvn sonar:sonar') + } + else if (env.CLOUD_PROVIDER == 'GCP') { + sh(script: "Quality Gate Failed !") + } + } + } + } +} diff --git a/src/com/meesho/stages/checkOut.groovy b/src/com/meesho/stages/checkOut.groovy new file mode 100644 index 0000000..5c5d0f3 --- /dev/null +++ b/src/com/meesho/stages/checkOut.groovy @@ -0,0 +1,45 @@ +package com.meesho.stages + +def run(Map param) { + try { + stage(stageName('Checkout from git')) { + dir("${param.repo_name}") { + checkout scm + if (fileExists('.gitmodules')) { + log.info('Repository has submodules - running git submodule init and update --recursive') + def credId = env.GITHUB_CRED?.trim() + if (!credId || credId == 'null') credId = 'svc-devops-meesho' + withCredentials([gitUsernamePassword(credentialsId: credId, gitToolName: 'git-tool')]) { + sh 'git submodule init && git submodule update --recursive' + } + } + } + } + } + catch( Exception e) { + env.msg = 'Error cloning the job. Error: ' + e.toString() + env.error_msg_to_db = 'Error cloning the job' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +def chekoutSubmodule(String repo_name) { + try { + def credId = env.GITHUB_CRED?.trim() + if (!credId || credId == 'null') credId = 'svc-devops-meesho' + dir(repo_name) { + withCredentials([gitUsernamePassword(credentialsId: credId, gitToolName: 'git-tool')]) { + sh(script:'git submodule init && git submodule update && git submodule update --recursive --remote') + } + } + } + catch(Exception e) { + env.msg = 'Error submodule checkout' + env.error_msg_to_db = env.msg + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} diff --git a/src/com/meesho/stages/deployArgoCD.groovy b/src/com/meesho/stages/deployArgoCD.groovy new file mode 100644 index 0000000..279abc2 --- /dev/null +++ b/src/com/meesho/stages/deployArgoCD.groovy @@ -0,0 +1,906 @@ +package com.meesho.stages + +import com.cwctravel.hudson.plugins.extended_choice_parameter.ExtendedChoiceParameterDefinition + +import com.meesho.utilities.getYamlParameter +import com.meesho.utilities.buTeamMapping +import com.meesho.utilities.gitActions +import com.meesho.utilities.constructTemplate +import com.meesho.utilities.constructParam +import com.meesho.utilities.getDockerParams +import com.meesho.utilities.nodePoolSelection + +def run(String repo_name, def deployment_order, def tag, def build_team, def dockerBuildVersion, String notify_channel) { + def userInput = '' + // def helm_repo_name = 'devops-helm-charts' + // def argo_repo_name = 'devops-argo-config' + + def gitObj = new gitActions() + def constructParam = new constructParam() + def yamlObj = new getYamlParameter() + def branch_name = env.BRANCH_NAME + def branch_param_map = ['(master|main|gcp-main|gcp-master|farmiso-main)': ['branch':'main', 'envrn':'prd'], + '(develop|gcp-dev)':['branch':'develop', 'envrn':'stg']] + + if (env.CHANGE_ID) { + branch_param_map = [ + '(develop|gcp-dev)':['branch':'feature', 'envrn':'ftr'], + '(master|main|gcp-main|gcp-master|farmiso-main)':['branch':'pre-prod', 'envrn':'int']] + branch_name = env.CHANGE_TARGET + } + def branch_param = branch_param_map.collectEntries { key, value -> branch_name.matches(key) ? value : [ : ] } + if (branch_param == [:]) { + branch_param = ['branch':'feature', 'envrn':'ftr'] + } + def helm_branch_name = branch_param.branch + def argo_branch_name = branch_param.branch + def envrn = branch_param.envrn + if (env.CHANGE_ID && env.CHANGE_BRANCH != "develop" && env.CHANGE_TARGET == "main" ) { + def allowedNonDevelopPrDeploymentToInt = constructParam.allowedNonDevelopPrDeploymentToIntRepos(repo_name) + if (!allowedNonDevelopPrDeploymentToInt) { + log.error("******** ONLY DEVELOP BRANCH PR ALLOWED FOR PRE-PROD ENV DEPLOYMENT ********") + return + } + } + log.info('Please provide apps to deploy') + timeout(unit: 'SECONDS', time: 300) { + userInput = wait_for_user_input(deployment_order) + } + + // change deployment order according to user input + + if (userInput == '') { + log.info('No Deployments selected. Running remaining steps.') + return + } + else if (!userInput.contains('All')) { + deployment_order = userInput.split(',') as List + } + + commit_branch_name = env.CHANGE_ID ? env.CHANGE_BRANCH : env.BRANCH_NAME + def commit_id = gitObj.fetchLatestCommitId(repo_name, commit_branch_name) + env.COMMIT_ID = commit_id + + env.SERVICES = deployment_order + //Show selected applications to deploy + log.info('Following services will be deployed:\n' + deployment_order.join('\n')) + // Clone the repo just once for all the deployments + gitObj.clone("${WORKSPACE}", "${env.helm_repo_name}", "${helm_branch_name}") + + // Clone the repo just once for all the deployments + gitObj.clone("${WORKSPACE}", "${env.argo_repo_name}", "${argo_branch_name}") + + for (deployment in deployment_order) { + try { + + def isMultizoneEnabled = constructParam.isMultizoneEnabled(deployment) + log.info(" Multi-zone enabled for ${deployment} - ${isMultizoneEnabled}") + if (isMultizoneEnabled) { + def multizone = "Multi-zone enabled for this deployable ${deployment}. Please use Ringmaster for deployment." + currentBuild.result = env.FAILURE + throw new Exception(multizone) + } + def value_binding = yamlObj.getParam("${repo_name}/deployments", "${deployment}.yaml") + constructParam.perDeploymentVars(value_binding) + // if(envrn == 'prd') { + // stage("${deployment}: Downscale Pods in Preprod Env") { + // preprod_downscale(repo_name, deployment,notify_channel) + // } + // } + stage("${deployment}: Update Argo App") { + update_argo_repo(repo_name, deployment, argo_branch_name, envrn) + } + stage('Update Argo App of apps') { + refresh_app_of_apps(envrn) + } + stage("${deployment}: Update Helm Repo") { + update_helm_repo(repo_name, deployment, tag, build_team, helm_branch_name, envrn, dockerBuildVersion, notify_channel) + } + stage("${deployment}: Refresh & Sync App in argoCD") { + refresh_and_sync(repo_name, deployment, envrn) + } + } + catch (Exception e) { + log.error(e) + currentBuild.result = env.FAILURE + } + } +} + +// def preprod_downscale(String repo_name, String deployment, String notify_channel){ +// def yamlobj = new getYamlParameter() +// def deploy_values = yamlobj.getParam("${repo_name}/deployments","${deployment}.yaml") +// def app_name = deploy_values.app_name +// def bu = deploy_values.bu +// def slack_channel = notify_channel +// def as_enabled = deploy_values.environment.'int'.as_enabled +// log.info("########################### Invoking Jenkins Job to Downscale Pods in Preprod Environment. ###########################") +// build wait: false, job: 'downscale-preprod-eks', parameters:[string(name:'app_name', value:"${app_name}"), +// string(name:'bu',value:"${bu}"), +// string(name:'as_enabled',value:"${as_enabled}"), +// string(name:'slack_channel',value:"${slack_channel}")] +// } + +def wait_for_user_input(def deployments) { + def userInput = '' + String choices = 'All,' + deployments.join(',') + int visibleItemCount = 1 + deployments.size() + def multiSelect = new ExtendedChoiceParameterDefinition('deployments', //name + 'PT_CHECKBOX', // parameter type + choices, //values + '', //projectName + '', //propertyFile + '', //groovyScript + '', //groovyScriptFile + '', //bindings + '', //groovyClasspath + '', //propertyKey + '', //defaultValue + '', //defaultPropertyFile + '', //defaultGroovyScript + '', //defaultGroovyScriptFile + '', //defaultBindings + '', //defaultGroovyClasspath + '', //defaultPropertyKey + '', //descriptionPropertyValue + '', //descriptionPropertyFile + '', //descriptionGroovyScript + '', //descriptionGroovyScriptFile + '', //descriptionBindings + '', //descriptionGroovyClasspath + '', //descriptionPropertyKey + '', //javascriptFile + '', //javascript + false, //saveJSONParameterToFile + false, //quoteValue + visibleItemCount, //visibleItemCount + 'Choose Deployments', //description + ',') //multiSelectDelimiter + + stage('wait for user input') { + try { + echo "Skipping User Input - ${env.skip_user_input}" + if (env.skip_user_input.toBoolean()) { + userInput = 'All' + } + else { + userInput = input message: 'Choose applications to deploy', ok: 'Deploy', parameters: [multiSelect] + } + return userInput + } + catch (Exception e) { + env.msg = 'Error in taking userInput' + e.toString() + env.error_msg_to_db = 'Error Taking User Input for Deployment of Applications' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + } +} + +def update_helm_repo(String repo_name, String deployment, String tag, String build_team, String helm_branch_name, String envrn, def dockerBuildVersion, String notify_channel) { + def yamlobj = new getYamlParameter() + def btObj = new buTeamMapping() + def gitObj = new gitActions() + def templateObj = new constructTemplate() + def nodePoolSelection = new nodePoolSelection() + // def helm_repo_name = 'devops-helm-charts' + + def value_yaml_file = get_value_yaml_file(dockerBuildVersion) + def value_binding1 = yamlobj.getParam("${repo_name}/deployments", "${deployment}.yaml") + if (value_binding1.containsKey('cron')) { + if (value_binding1['cron']) { + value_yaml_file = 'cron-values.yaml' + } + } + + def env_norm = ['prd':'prod', 'int':'pre-prod', 'stg':'stg', 'ftr':'feature'] + def pr_num = '' + def commit_status = 0 + + try { + // Get values from deployments/deployment.yaml + isfeatureDeployment = (envrn == 'ftr') ? true : false + value_binding1 = yamlobj.getParam("${repo_name}/deployments", "${deployment}.yaml") + def memory_request = value_binding1.environment[env.cicd_environment].memory_request + def cpu_request = value_binding1.environment[env.cicd_environment].cpu_request + def priority_v2 = value_binding1.priority_v2 + def commit_id = env.COMMIT_ID + if ( env.CLOUD_PROVIDER == 'GCP' && !value_binding1.containsKey('cron') ) { + nodeSelectorValue = nodePoolSelection.run(memory_request, cpu_request.toString(),priority_v2) + echo "Node Selector Value is ${nodeSelectorValue}" + value_binding1['nodeSelectorValue'] = nodeSelectorValue + } + def xms = '' + def xmx = '' + if (!value_binding1.containsKey('cron')) { + if (dockerBuildVersion.contains('maven') || dockerBuildVersion.contains('gradle') || dockerBuildVersion.contains('go')) { + value_binding1['activeProcessorCount'] = calculate_active_processors(cpu_request.toString()) + } + if (dockerBuildVersion.contains('maven') || dockerBuildVersion.contains('gradle')) { + def memory_limit = value_binding1.environment[env.cicd_environment].memory_limit + def deployment_args = value_binding1.environment[env.cicd_environment].deployment_args + echo "memory_limit is ${memory_limit}" + def memory_string = memory_limit + echo "performing operations on this memory string - ${memory_string}" + + def memory_value = '' + + if ( memory_string.contains('M') ) { + memory_value = memory_string.replaceAll('Mi', '') + memory_value = memory_value.replaceAll('M', '') + try { + memory_value = memory_value.toInteger() + } + catch (NumberFormatException e) { + memory_value = memory_value.toDouble() + memory_value = memory_value.toInteger() + } + } + else if ( memory_string.contains('G') ) { + memory_value = memory_string.replaceAll('Gi', '') + memory_value = memory_value.replaceAll('G', '') + try { + memory_value = memory_value.toInteger() * 1024 + } + catch (NumberFormatException e) { + memory_value = memory_value.toDouble() * 1024 + memory_value = memory_value.toInteger() + } + } + + memory_value = memory_value * 0.5 + memory_value = memory_value.toInteger() + xms = "${memory_value}M" + xmx = "${memory_value}M" + + echo "xms and xmx from memory value - ${memory_value} are - ${xms} ${xmx}" + for (arg in deployment_args) { + if (arg.contains('Xms')) { + xms = arg.replaceAll('.*Xms', '') + echo "xms from deployment_args - ${xms}" + } + if (arg.contains('Xmx')) { + xmx = arg.replaceAll('.*Xmx', '') + echo "xmx from deployment_args - ${xmx}" + } + } + echo "final xms and xmx - ${xms} ${xmx}" + } + } + + value_binding1['dockerBuildVersion'] = dockerBuildVersion == 'python-3.10.12' ? 'python-3.7' : dockerBuildVersion + deployEnv = !value_binding1['environment'].containsKey(envrn) && isfeatureDeployment ? 'stg' : envrn + Map value_binding = value_binding1['environment'].collectEntries { key, value -> deployEnv.matches(key) ? value : [ : ] } + value_binding1.remove('environment') + value_binding1.putAll(value_binding) + + value_binding1['repo_name'] = repo_name + value_binding1['environment'] = envrn + value_binding1['environment_norm'] = env_norm[envrn] + value_binding1['tag'] = tag + value_binding1['repo_name'] = repo_name + value_binding1['build_team'] = build_team + value_binding1['commit_id'] = commit_id + + def buini = btObj.get_bu_initials(value_binding1.bu) + def bu = btObj.get_bu_initials(value_binding1.bu) + def teamini = btObj.get_team_initials(value_binding1.team) + def team = btObj.get_team_initials(value_binding1.team) + def app_name = value_binding1.app_name + def app_branch = app_name + '-' + helm_branch_name + + def app_helm_repo = "${env.helm_repo_name}/${env.helmChartsPath}/${buini}/${teamini}/${app_name}" + def ingress_val = value_binding1.ingress_val ?: (env.CHANGE_ID) ? "pr-${CHANGE_ID}" : "${BRANCH_NAME}" + + sh "chmod -R 777 ${app_helm_repo}" + + sh "yq . ${app_helm_repo}/values_properties.yaml -y > a.yaml;mv a.yaml ${app_helm_repo}/values_properties.yaml" + sh "cat ${app_helm_repo}/values_properties.yaml" + if (isfeatureDeployment) { + env.ingress_val = ingress_val + value_binding1['env_ns'] = ingress_val + value_binding1['vault_env'] = value_binding1['create_vault_path'] ? ingress_val : 'stg' + sh(returnStdout: true, script: """ + mkdir -p ${app_helm_repo}/${ingress_val} + sed "s/INGRESS_PR_NUMBER/${ingress_val}/g" ${app_helm_repo}/values_properties.yaml > ${app_helm_repo}/${ingress_val}/values_properties.yaml + """) + value_binding2 = yamlobj.getParam("${app_helm_repo}/${ingress_val}", 'values_properties.yaml') + app_branch = ingress_val + '-' + app_branch + } + else { + value_binding1['vault_env'] = envrn + value_binding1['env_ns'] = envrn + value_binding2 = yamlobj.getParam("${app_helm_repo}", 'values_properties.yaml') + echo '722 Printing Value Binding 2' + print value_binding2 + } + + value_binding1.putAll(value_binding2) + value_binding1['bu_norm'] = value_binding1['bu'] + value_binding1['team_norm'] = value_binding1['team'] + value_binding1['bu'] = buini + value_binding1['team'] = teamini + + + def serviceTypes = ["httpstateless", "consumer", "producer", "scheduler", "worker", "grpc", "web", "websocket", "cache", "database"] + //validate service type params + if (value_binding1['service_type']){ + // Validating the parameter type + if(!(value_binding1['service_type'] instanceof List) || value_binding1['service_type'].isEmpty()){ + env.msg = 'You have not specified service_type correctly. Exiting the pipeline.' + log.error(env.msg) + sh 'exit 1' + } + // Check if value_binding1['service_type'] contains any service type not in servicesType + def invalidServiceTypes = value_binding1['service_type'].findAll { !serviceTypes.contains(it) } + if (!invalidServiceTypes.isEmpty()) { + log.error("Invalid service type(s): ${invalidServiceTypes.join(',')}. Allowed service_types are: ${serviceTypes.join(',')}") + sh 'exit 1' + } + value_binding1['service_type_norm'] = value_binding1['service_type'].join(',') + } + else{ + value_binding1['service_type_norm']='' + } + + // Enable backward compatibility for missing keys + enable_backward_compatibility(value_binding1) + value_binding1['prismsdk_environment'] = 'PRODUCTION' // THIS KEY WILL CHANGE ONCE 3RD CONFIRM TO STANDERIZE + if (envrn == 'stg'){ + value_binding1['otel_enabled'] = true + value_binding1['prismsdk_environment'] = 'SANDBOX' // THIS KEY WILL CHANGE ONCE 3RD CONFIRM TO STANDERIZE + } + // Override hot fix and notify_channel + value_binding1['canary']['skipAnalysis'] = (env.hot_fix) ? true : value_binding1['canary']['skipAnalysis'] + value_binding1['canary']['slackChannel'] = notify_channel + value_binding1['xms'] = xms + value_binding1['xmx'] = xmx + value_binding1['CLOUD_PROVIDER'] = env.CLOUD_PROVIDER + def appConfig = value_binding1['appConfigEnabled'] + if (appConfig) { + def configModule = (value_binding1["module"] == 'module_less') ? repo_name : value_binding1["module"] + // read the static config from configs directory + def value_binding3 = yamlobj.getParamAsString("${repo_name}/configs/${configModule}", "application-${envrn}.yml") + // to update in values.yaml + value_binding1['staticAppConfigData'] = value_binding3 + //read the dynamic config from config directory + def value_binding4 = yamlobj.getParamAsString("${repo_name}/configs/${configModule}", "application-dyn-${envrn}.yml") + // to update in values.yaml + value_binding1['dynamicAppConfigData'] = value_binding4 + echo 'printing the values_binding3 and values_binding4' + print value_binding3 + print value_binding4 + value_binding1['vault_env'] = value_binding1['vault_env'] + '-cac' + + } + + + gitObj.preDeleteBranch(env.helm_repo_name, helm_branch_name, app_branch) + gitObj.branchCheckOut(env.helm_repo_name, app_branch) + + echo "Helm Step - Value Binding 1 - ${value_binding1}" + if (isfeatureDeployment) { + echo 'It is feature deployment' + templateObj.renderTemplate(value_binding1, value_yaml_file, "${app_helm_repo}/${ingress_val}/values.yaml") + echo 'Before YAML Linting' + sh "cat ${app_helm_repo}/${ingress_val}/values.yaml" + sh "yq . ${app_helm_repo}/${ingress_val}/values.yaml -y > a.yaml;mv a.yaml ${app_helm_repo}/${ingress_val}/values.yaml" + echo 'After YAML Linting' + sh "cat ${app_helm_repo}/${ingress_val}/values.yaml" + gitObj.add(env.helm_repo_name, "${env.helmChartsPath}/${buini}/${teamini}/${app_name}/${ingress_val}/values.yaml") + } + else { + echo 'Not a feature deployment' + templateObj.renderTemplate(value_binding1, value_yaml_file, "${app_helm_repo}/values.yaml") + echo 'Before YAML Linting' + sh "cat ${app_helm_repo}/values.yaml" + + // Perform YAML linting + sh "yq . ${app_helm_repo}/values.yaml -y > a.yaml; mv a.yaml ${app_helm_repo}/values.yaml" + echo 'After YAML Linting' + sh "cat ${app_helm_repo}/values.yaml" + + // creating a map from final values.yaml to check canary enforcement conditions + def valuesMap = yamlobj.getParam("${app_helm_repo}", "values.yaml") + + // Canary enforcement for sp0 services + // Also will have to check for cron, worker and scheduler services + def enforceCanary = false + if (!app_helm_repo.contains("cron") && !app_helm_repo.contains("worker") && !app_helm_repo.contains("scheduler") && !app_helm_repo.contains("consumer") && (valuesMap["labels"]["priority_v2"] == "sp0" || valuesMap["labels"]["priority_v2"] == "up0") && !value_binding1["addHeadless"] && !value_binding1['dockerBuildVersion'].contains("node") && envrn == "prd") { + enforceCanary = true + } + + def proceed = false + def deploymentFailureErrorMessage + if (enforceCanary) { + if (valuesMap["canary"]["enabled"] == true) { + if (valuesMap["canary"]["skipAnalysis"] == false) { + if (valuesMap["canary"]["enableManualPromotion"] == true) { + proceed = true + } else { + error("Error: Update the enableManualPromotion parameter") + } + } else { + error("Error: Update the skipAnalysis parameter") + + } + } else { + error("Error: Enable canary and retry") + } + } else { + proceed = true + } + + // Check if priority labels and environment conditions require critical dependency check + def criticalPriorities = ["sp0", "up0", "cp0", "sp1", "up1", "cp1"] + def priority = valuesMap["labels"]["priority_v2"] + if (criticalPriorities.contains(priority) && (envrn == "prd")) { + isDependabotCritcal = dependabotCriticalCheck(repo_name) + if (isDependabotCritcal) { + error("Error: Critical vulnerabilities found in repo: " + repo_name + " \nPlease resolve the alerts marked with CRITICAL here and retry: https://github.com/Meesho/" + repo_name + "/security/dependabot and Retry.") + } + } + + def param = new constructParam() + log.info("checking if appConfig is enabled") + def isAppConfigDisabled = param.appConfigDisabledForbidden(appConfig, repo_name, envrn, dockerBuildVersion) + if (isAppConfigDisabled){ + error("Error: appConfig is disabled , onboard your application with config-as-code changes") + } + + sh "cat ${app_helm_repo}/values.yaml" + gitObj.add(env.helm_repo_name, "${env.helmChartsPath}/${buini}/${teamini}/${app_name}/values.yaml") + + } + + + commit_status = gitObj.codeCommit(env.helm_repo_name, app_branch, 'Generating values yaml file') + if (commit_status == 0) { + gitObj.codePush(env.helm_repo_name, app_branch) + pr_num = gitObj.createPR(app_name, env.helm_repo_name, helm_branch_name, app_branch, 'Merge helm values file') + gitObj.mergePR(env.helm_repo_name, pr_num, app_branch) + gitObj.deleteBranch(env.helm_repo_name, helm_branch_name, app_branch) + } + } + catch (FileNotFoundException e) { + env.msg = 'Error Updating Helm Repo ' + e.toString() + env.error_msg_to_db = 'File Not Found' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + catch (Exception e) { + env.msg = 'Error Updating Helm Repo ' + e.toString() + env.error_msg_to_db += 'Error Updating in Helm repo. Git error message - ' + env.error_part_msg_to_db + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +def refresh_and_sync(String repo_name, String deployment, String envrn) { + def yamlobj = new getYamlParameter() + def deploy_values = yamlobj.getParam("${repo_name}/deployments", "${deployment}.yaml") + def app_name = deploy_values.app_name + // def argoUrl = (env.BRANCH_NAME == 'master' || env.BRANCH_NAME == 'main' || env.CHANGE_TARGET == 'master' || env.CHANGE_TARGET == 'main') ? 'prod-ops-argocd.meesho.com' : 'stg-dev-argocd.meeshotest.in' + def argoEnv = (envrn == 'ftr') ? env.ingress_val : envrn + + log.info('########################### Pulling latest changes in ArgoCD. ###########################') + withCredentials([usernamePassword(credentialsId: env.argoCreds, passwordVariable: 'ARGO_PASSWORD', usernameVariable: 'ARGO_USERNAME')]) { + try { + sh """ + set +x + argocd login ${env.argoURL}:443 --username ${ARGO_USERNAME} --password ${ARGO_PASSWORD} --grpc-web + argocd app get --hard-refresh ${argoEnv}-${app_name} --grpc-web + """ + } + catch (Exception e) { //added try catch block here + env.msg = "Error in hard refresh of app ${app_name} full error: ${e}" + env.error_msg_to_db += "Error in hard refresh of app ${app_name};" + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } + log.info('########################### Syncing latest changes in ArgoCD. ###########################') + return_value = sh(returnStatus: true, script: "argocd app sync ${argoEnv}-${app_name} --grpc-web --http-retry-max 3 --retry-backoff-duration 1m") as Integer + if (return_value == 0) { + log.info('App synced succesfully.') + } else { + env.msg = 'App sync failed. Please check in ArgoCD UI.' + env.error_msg_to_db += "Error Argo App sync failed ${app_name};" + log.error(env.msg) + error "${env.msg}" + } + } +} + +def refresh_app_of_apps(String envrn) { + try { + appofapps = env.argoIncubator + log.info('########################### Pulling latest changes in ArgoCD for App of Apps. ###########################') + withCredentials([usernamePassword(credentialsId: env.argoCreds, passwordVariable: 'ARGO_PASSWORD', usernameVariable: 'ARGO_USERNAME')]) { + sh """ + set +x + argocd login ${env.argoURL}:443 --username ${ARGO_USERNAME} --password ${ARGO_PASSWORD} --grpc-web + argocd app sync ${appofapps} --grpc-web --http-retry-max 3 --retry-backoff-duration 1m || true + """ + } + } + catch (Exception e) { + env.msg = 'Error in App Sync' + e.toString() + env.error_msg_to_db += 'Error syncing app of app;' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} + +def update_argo_repo(String repo_name, String deployment, String argoBranch, String envrn) { + def yamlobj = new getYamlParameter() + def btObj = new buTeamMapping() + def gitObj = new gitActions() + def templateObj = new constructTemplate() + // def argo_repo_name = 'devops-argo-config' + def pr_num = '' + def commit_status = 0 + def commit_id = env.COMMIT_ID + + value_binding1 = yamlobj.getParam("${repo_name}/deployments", "${deployment}.yaml") + value_binding1['CLOUD_PROVIDER'] = env.CLOUD_PROVIDER + isfeatureDeployment = (envrn == 'ftr') ? true : false + deployEnv = !value_binding1['environment'].containsKey(envrn) && isfeatureDeployment ? 'stg' : envrn + Map value_binding = value_binding1['environment'].collectEntries { key, value -> deployEnv.matches(key) ? value : [ : ] } + value_binding1.remove('environment') + value_binding1.putAll(value_binding) + value_binding1['environment'] = envrn + + def buini = btObj.get_bu_initials(value_binding1.bu) + def bu = value_binding1.bu + + def teamini = btObj.get_team_initials(value_binding1.team) + def team = value_binding1.team + def app_name = value_binding1.app_name + def filename = "${teamini}-${app_name}.yaml" + def helm_values_path = "${env.helmChartsPath}/${buini}/${teamini}/${app_name}" + def app_branch = app_name + '-' + argoBranch + def ingress_val = value_binding1.ingress_val ?: (env.CHANGE_ID) ? "pr-${CHANGE_ID}" : "${BRANCH_NAME}" + value_binding1['helm_version'] = value_binding1.helm_version ?: env.defaultHelmChartVersion + + if (isfeatureDeployment) { + value_binding1['env_ns'] = ingress_val + value_binding1['helm_values_path'] = "${env.helmChartsPath}/${buini}/${teamini}/${app_name}/${ingress_val}" + filename = ingress_val + '-' + "${app_name}.yaml" + app_branch = ingress_val + '-' + app_branch + } + else { + value_binding1['env_ns'] = envrn + value_binding1['helm_values_path'] = "${env.helmChartsPath}/${buini}/${teamini}/${app_name}" + filename = "${teamini}-${app_name}.yaml" + } + + value_binding1['server'] = env.clusterName + value_binding1['clusterName'] = env.clusterName + + value_binding1['branch_name'] = argoBranch + value_binding1['buini'] = buini + value_binding1['teamini'] = teamini + + value_binding1['priority_v2'] = value_binding1.priority_v2 ?: 'cp3' + value_binding1['primary_owner'] = value_binding1.primary_owner != null ? value_binding1.primary_owner.split('@')[0] : value_binding1.team + value_binding1['secondary_owner'] = value_binding1.secondary_owner != null ? value_binding1.secondary_owner.split('@')[0] : value_binding1.team + value_binding1['argoAppNS'] = env.argoAppNS + value_binding1['commit_id'] = commit_id + log.info(commit_id) + gitObj.preDeleteBranch(env.argo_repo_name, argoBranch, app_branch) + gitObj.branchCheckOut(env.argo_repo_name, app_branch) + echo "Argo Step - Value Binding 1 - ${value_binding1}" + + dir(env.argo_repo_name) { + def dirExists = sh(script: "cat ${env.argoAppsPath}/${filename}", returnStatus: true) + if ( dirExists != 0 ) { + sh "mkdir -p ${env.argoAppsPath}" + sh "touch ${env.argoAppsPath}/${filename}" + } + sh "chmod -R 777 ${env.argoAppsPath}/${filename}" + templateObj.renderTemplate(value_binding1, 'argoApp.yaml', "${env.argoAppsPath}/${filename}") + sh "yq . ${env.argoAppsPath}/${filename} -y > a.yaml;mv a.yaml ${env.argoAppsPath}/${filename}" + sh "cat ${env.argoAppsPath}/${filename}" + } + + gitObj.add(env.argo_repo_name, "${env.argoAppsPath}/${filename}") + commit_status = gitObj.codeCommit(env.argo_repo_name, app_branch, "onboarding ${deployment} app to ARGO") + if (commit_status == 0) { + gitObj.codePush(env.argo_repo_name, app_branch) + pr_num = gitObj.createPR(app_name, env.argo_repo_name, argoBranch, app_branch, 'Merge argo application configuration') + gitObj.mergePR(env.argo_repo_name, pr_num, app_branch) + gitObj.deleteBranch(env.argo_repo_name, argoBranch, app_branch) + } +} + +def calculate_active_processors(String cpu_request) { + // If lowercase m is present + if (cpu_request.contains('m')) { + // Remove m + cpu_request = cpu_request.replaceAll('m', '') + + // If this fails means invalid input is given with m, so we can let the pipeline fail here + // Convert to double, divide by thousand to get another double, round it off and then convert to integer. + return Math.ceil(cpu_request.toDouble() / 1000).toInteger() + } + + return Math.ceil(cpu_request.toDouble()).toInteger() +} + +def enable_backward_compatibility(def bindings) { + // def primaryEmailInitial = bindings.primary_owner.tokenize( '@' )[0]: null + // def secondaryEmailInitial = bindings.secondary_owner.tokenize( '@' )[0]: null + def dparam_obj = new getDockerParams() + bindings['registry'] = env.registry + def canary_default = [ + 'progressDeadlineSeconds': 300, + 'analysisInterval': '120s', + 'analysisThreshold': 5, + 'analysisMaxWeight': 5, + 'analysisStepWeight': 5, + 'analysisMetrics':[ + 'thresholdRangeMin': 0.99, + 'interval': '1m'], + 'skipAnalysis': false + ] + def statefulset_default = [ + 'updateStrategy': 'RollingUpdate', + 'volumeType': 'dynamic', + 'dynamicVolume':[ + 'accessMode': 'ReadWriteMany', + 'mountPath': '/opt/data', + 'size': '5Gi', + 'storageClass': ''], + 'staticVolume':[ + 'accessMode': 'ReadWriteMany', + 'size': '5Gi', + 'mountPath': '/opt/data', + 'storageClass': '', + 'volumeHandle': '', + 'csiDriver': ''] + ] + bindings['hostAliases'] = bindings.hostAliases ?: false + if (env.CLOUD_PROVIDER == 'AWS') { + bindings['nodeSelector'] = bindings.nodeSelector ?: (env.cicd_environment == 'stg' || env.cicd_environment == 'ftr') ? bindings.bu : ( env.cicd_environment == 'int' ? bindings.bu + '-int' : bindings.team ) + bindings['nodeSelectorValue'] = 'dedicated' + } + else if (env.CLOUD_PROVIDER == 'GCP') { + if (env.cicd_environment == 'int') { + bindings['nodeSelector'] = bindings.nodeSelector ?: 'cloud.google.com/compute-class' + } else { + bindings['nodeSelector'] = bindings.nodeSelector ?: 'dedicated' + } + bindings['nodeSelectorValue'] = bindings.nodeSelectorValue ?: 'megatetra' + } + bindings['triggers'] = bindings.triggers ?: false + bindings['host'] = bindings.host ?: false + bindings['hosts'] = bindings.hosts ?: false + bindings['grpc_host'] = bindings.grpc_host ?: false + bindings['grpc_hosts'] = bindings.grpc_hosts ?: false + bindings['serviceAccount'] = bindings.serviceAccount ?: false + bindings['ingress_annotations'] = bindings.ingress_annotations ?: '' + bindings['canary'] = bindings.canary ?: canary_default + bindings['minCanaryReplicas'] = bindings.canary?.minCanaryReplicas ?: bindings.minCanaryReplicas ?: bindings.as_min + bindings['maxCanaryReplicas'] = bindings.canary?.maxCanaryReplicas ?: bindings.maxCanaryReplicas ?: bindings.as_max + bindings['enableManualPromotion'] = bindings.canary?.enableManualPromotion ?: bindings.enableManualPromotion ?: false + bindings['cpu_limit'] = bindings.cpu_limit ?: bindings.cpu_request + bindings['slowStartWindow'] = bindings.slowStartWindow ?: false + bindings['slowStartAggression'] = bindings.slowStartAggression ?: '1' + bindings['slowStartMinPercent'] = bindings.slowStartMinPercent ?: '10' + bindings['lifecycle'] = bindings.lifecycle ?: false + bindings['maxSurge'] = bindings.maxSurge ?: '50' + bindings['as_down_pod_count'] = bindings.as_down_pod_count ?: '2' + bindings['as_up_pod_count'] = bindings.as_up_pod_count ?: '2' + bindings['as_up_pod_percentage'] = bindings.as_up_pod_percentage ?: '10' + bindings['createContourGateway'] = bindings.createContourGateway ?: false + bindings['service_annotations'] = bindings.service_annotations ?: false + bindings['pod_annotations'] = bindings.pod_annotations ?: false + bindings['kind'] = bindings.kind ?: 'deployment' + bindings['statefulset'] = bindings.statefulset ?: statefulset_default + bindings['contourResponseTimeout'] = bindings.contourResponseTimeout ?: false + bindings['pdbMinAvailable'] = bindings.pdbMinAvailable ?: '' + bindings['pdbMaxUnavailable'] = bindings.pdbMaxUnavailable ?: '10%' + + // Just to keep compatibility for services which still use grpc_port + // If someone has supplied primary_port, then it is used + // Else we check for grpc_port, and that is used + // If none of the above is supplied, then app_port is used just like normal flow + bindings['primary_port'] = bindings.primary_port ?: bindings.grpc_port ?: bindings.app_port + bindings['grpc_port'] = bindings.grpc_port ?: false + // bindings['xmx'] = bindings.xmx ?: '50.0' + // bindings['xms'] = bindings.xms ?: '50.0' + bindings['enableWebsocket'] = bindings.enableWebsocket ?: false + bindings['external_secrets_annotations'] = bindings.external_secrets_annotations ?: '' + bindings['liveness_failure_threshold'] = bindings.liveness_failure_threshold ?: '5' + bindings['liveness_period_seconds'] = bindings.liveness_period_seconds ?: bindings.team_norm == 'ml-platform' ? '5' : '10' + bindings['liveness_success_threshold'] = bindings.liveness_success_threshold ?: '1' + bindings['liveness_timeout_seconds'] = bindings.liveness_timeout_seconds ?: '2' + bindings['readiness_failure_threshold'] = bindings.readiness_failure_threshold ?: '5' + bindings['readiness_period_seconds'] = bindings.liveness_period_seconds ?: bindings.team_norm == 'ml-platform' ? '5' : '10' + bindings['readiness_success_threshold'] = bindings.liveness_success_threshold ?: '1' + bindings['readiness_timeout_seconds'] = bindings.liveness_timeout_seconds ?: '2' + bindings['addon_ports'] = bindings.addon_ports ?: false + + validateRequiredMetadata(bindings) + + bindings['priority_v2'] = bindings.priority_v2 ?: 'cp3' + bindings['primary_owner'] = bindings.primary_owner != null ? bindings.primary_owner.split('@')[0] : bindings.team_norm + bindings['secondary_owner'] = bindings.secondary_owner != null ? bindings.secondary_owner.split('@')[0] : bindings.team_norm + if (env.CLOUD_PROVIDER == 'AWS') { + if (bindings.otel_enabled == null) { + bindings['otel_enabled'] = (bindings.priority_v2 == 'cp1' || bindings.priority_v2 == 'up1') ? true : false + } + else { + bindings['otel_enabled'] = bindings.otel_enabled + } + } + else if (env.CLOUD_PROVIDER == 'GCP') { + echo "Checking for otel value - ${bindings.otel_enabled}" + if (bindings.otel_enabled == null) { + bindings['otel_enabled'] = true + } + else { + bindings['otel_enabled'] = bindings.otel_enabled + } + + // Setting metrics mode -> avaliable options: telegraf | otel | dual + echo "Setting metrics mode - ${bindings.metrics_mode}" + // Setting default values + bindings['telegraf_metrics'] = false + bindings['otel_metrics'] = false + bindings['metrics_mode'] = bindings.metrics_mode?.toLowerCase() + def samplerArg = bindings.otel_traces_sampler_arg + def isValidSamplerArg = false + if (samplerArg != null && (samplerArg instanceof Float || samplerArg instanceof Double)) { + if (samplerArg >= 0.0 && samplerArg <= 1.0) { + isValidSamplerArg = true + } + } + bindings['otel_traces_sampler_arg'] = isValidSamplerArg ? samplerArg : '0.1' + log.info("1 - otel_traces_sampler_arg - ${bindings.otel_traces_sampler_arg}") + log.info("2 - ${bindings['otel_traces_sampler_arg']}") + // Enabling metrics based on mode + if (bindings.metrics_mode == 'telegraf') { + bindings['telegraf_metrics'] = true + } + else if (bindings.metrics_mode == 'otel') { + bindings['otel_metrics'] = true + } + else if (bindings.metrics_mode == 'dual') { + bindings['telegraf_metrics'] = true + bindings['otel_metrics'] = true + } + else { + bindings['metrics_mode'] = 'telegraf' + bindings['telegraf_metrics'] = true + } + } + bindings['command'] = bindings.command + if (bindings.command == null) { + bindings['command'] = get_default_command(bindings['dockerBuildVersion']) + } + bindings['as_down_stable_window'] = bindings.as_down_stable_window ?: '1800' + bindings['podDistributionSkew'] = bindings.podDistributionSkew ?: false + bindings["appConfigEnabled"] = bindings.appConfigEnabled ?: false + bindings["addHeadless"] = bindings.addHeadless?: false +} + +def get_value_yaml_file(def dockerBuildVersion) { + switch (dockerBuildVersion) { + case ~/^maven-.*/: return 'values.yaml' + case ~/^node-.*/: return 'node-values.yaml' + case ~/^python-.*/: return 'python-values.yaml' + case ~/^go.*/: return 'go-values.yaml' + case 'php': return 'php-values.yaml' + case 'gradle': return 'values.yaml' + } +} + +def get_default_command(def dockerBuildVersion) { + switch (dockerBuildVersion) { + case ~/^maven-.*/: return 'java' + case ~/^node-.*/: return 'pm2-runtime' + case ~/^go.*/: return '/app/server' + case 'php': return 'apache2-foreground' + case 'gradle': return 'java' + default: return null + } +} + +def validateRequiredMetadata(def bindings) { + def requiredParams = ['primary_owner', 'secondary_owner', 'priority_v2', 'service_type'] + def serviceOwners = ['primary_owner', 'secondary_owner'] + // Validate if required parameters are present + for (param in requiredParams) { + if (bindings[param] == null || bindings[param] == '') { + env.msg = 'You have not supplied ' + param + '. Exiting the pipeline.' + log.error(env.msg) + sh 'exit 1' + } + } + + // Validate if the owners are valid or not + if (env.cicd_environment == 'dev' || env.cicd_environment == 'ftr' || env.cicd_environment == 'stg') { + return + } + + for (param in serviceOwners) { + final String owner = bindings[param] + final String url = "https://pulse.meeshogcp.in/api/anonymous-User/userexist?email=${owner}" + final def(String response, String code) = sh(returnStdout: true, script: """ + set +x + curl -s -X GET -w '\n%{response_code}' $url + set -x + """).trim().tokenize('\n') + + if (code != "200") { + // Let's not break the pipeline in case the API fails + log.info("Received ${code} code from Pulse while checking for user. Skipping the checks further and letting the pipeline proceed.") + } else { + // Check if the user exists + def jqCommand = "echo '${response}' | jq -r '.exists'" + def userExists = sh(returnStdout: true, script: """ + set +x + ${jqCommand} + set -x + """).trim() + if (userExists != "true") { + env.msg = "Invalid value provided in ${param}. Check if the user ${owner} exists" + log.error(env.msg) + sh 'exit 1' + } + } + } +} + +def dependabotCriticalCheck(def repoName) { + def repo = repoName + + def alerts + withCredentials([usernamePassword(credentialsId: "${env.GITHUB_CRED}", usernameVariable:'user', passwordVariable: 'token')]) { + validateDependabot = httpRequest httpMode: 'GET', + customHeaders: [ + [name: 'Accept', value: 'application/vnd.github+json'], + [maskValue: true, name: 'Authorization', value: 'Bearer ' + token] + ], + url: "https://api.github.com/repos/meesho/${repo}/dependabot/alerts?state=open&per_page=100", + validResponseCodes: '200', + timeout: 10 + alerts = readJSON(text: validateDependabot.content) + + } + + // if (!alerts) { + // println "Failed to fetch alerts for ${repo}." + // return false + // } + + // Parse the JSON response + if (!alerts) { + println "No alerts found for ${repo}." + return false + } + + def criticalCount = 0 + alerts.each { alert -> + def severity = alert?.security_vulnerability?.severity + + if (severity == "critical") { + criticalCount++ + } + + } + + if (criticalCount > 0) { + println "Critical vulnerabilities found in ${repo}: ${criticalCount}" + return true // Critical vulnerabilities found + } else { + println "No critical vulnerabilities in ${repo}." + return false // No critical vulnerabilities found + } +} diff --git a/src/com/meesho/stages/deployJar.groovy b/src/com/meesho/stages/deployJar.groovy new file mode 100644 index 0000000..9946abf --- /dev/null +++ b/src/com/meesho/stages/deployJar.groovy @@ -0,0 +1,32 @@ +package com.meesho.stages + +def run(Map param){ + //get the jar for deployment, get it from the params module + //run the ansible playbook + //create the inventory file with given ip in parameters + def server_ip = param.run_automation.server_ip + def app_name = param.run_automation.app_name + def healthcheck_api = param.run_automation.healthcheck_api + def app_port = param.run_automation.app_port + def repo_name = param.repo_name + + deploy(server_ip,repo_name,app_name,healthcheck_api,app_port) +} + +def deploy(String server_ip, String repo_name, String app_name, String healthcheck_api, String app_port){ + try{ + stage('Deploying JAR'){ + sh "echo '$server_ip' > host_file.txt" + echo "inventory created" + def playbook_content = libraryResource 'com/meesho/deployJar.yaml' + writeFile file:"deployJar.yaml", text: playbook_content + sh(returnStatus: true, script: "ansible-playbook -i host_file.txt -u 'ubuntu' -e 'env=stage' -e 'app_name=${app_name}' -e 'repo_name=${repo_name}' -e 'healthcheck_api=${healthcheck_api}' -e 'app_port=${app_port}' deployJar.yaml -v") + } + } + catch( Exception e) { + env.msg = "Error while deploying JAR. Please check console output for more details." + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} diff --git a/src/com/meesho/stages/deployRingmaster.groovy b/src/com/meesho/stages/deployRingmaster.groovy new file mode 100644 index 0000000..720f1a8 --- /dev/null +++ b/src/com/meesho/stages/deployRingmaster.groovy @@ -0,0 +1,144 @@ +package com.meesho.stages + +import com.meesho.utilities.getYamlParameter +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatterBuilder + +def run(String repo_name, def deployment_order, def tag, def build_team, def dockerBuildVersion, String notify_channel){ + def yamlobj = new getYamlParameter() + def applicationNames = [] + def jobStatusMap=["SUCCESS": "BUILD_STATUS_COMPLETED","FAILURE": "BUILD_STATUS_FAILED","UNSTABLE":"BUILD_STATUS_FAILED"] + def build_user = currentBuild.rawBuild.getCause(Cause.UserIdCause).getUserId() + + for (deployment in deployment_order){ + def value_binding = yamlobj.getParam("${repo_name}/deployments","${deployment}.yaml") + applicationNames.add(value_binding['app_name']) + } + def end_time = getDateTime() + def working_env = (env.CLOUD_PROVIDER == "GCP")? "gcp_${env.cicd_environment}" : env.cicd_environment + + def jsonMap = [:] + jsonMap["hot_fix"] = env.hot_fix ? true : false + jsonMap["job_name"] = env.JOB_NAME ? env.JOB_NAME.split('/')[0] : "unknown" + jsonMap["build_no"] = env.BUILD_NUMBER + jsonMap["image"] = tag + jsonMap["applications"] = applicationNames + jsonMap["job_status"] = jobStatusMap[currentBuild?.currentResult] + jsonMap["err_msg"] = env.error_msg_to_db + jsonMap["end_time"] = end_time + jsonMap["email"] = currentBuild?.rawBuild?.getCause(Cause.UserIdCause)?.getUserId() + jsonMap["start_time"] = env.STARTTIME + jsonMap["build_team"] = build_team + jsonMap["docker_build_version"] = dockerBuildVersion + jsonMap["notify_channel"] = notify_channel + jsonMap["branch_name"] = env.CHANGE_ID ? env.CHANGE_BRANCH : env.BRANCH_NAME + jsonMap["commit_id"] = env.commit_id + jsonMap["deploy_argo"] = env.deployArgo != null ? env.deployArgo.toBoolean() : false + jsonMap["pr_number"] = env.CHANGE_ID ? env.CHANGE_ID.toInteger() : 0 + + final String baseUrl + + switch (env.cicd_environment) { + case 'prd': + case 'int': + baseUrl = 'https://ringmaster-api.meeshogcp.in/api/v1/key/cicd/cd/update' + break + default: + baseUrl = 'https://ringmaster-api.admin.meeshogcp.in/api/v1/key/cicd/cd/update' + } + final String url = "${baseUrl}?workingEnv=${working_env}" + final String header = "Content-Type: application/json" + final String jsonData = writeJSON returnText: true, json: jsonMap + + + if (build_user == "ringmaster-bot"){ + callApi(url, header, jsonData) + } else{ + def newCICD_Payload = [:] + newCICD_Payload["repo_name"] = repo_name + newCICD_Payload["source_branch"] = env.CHANGE_ID ? env.CHANGE_BRANCH : env.BRANCH_NAME + newCICD_Payload["pull_request_number"] = env.CHANGE_ID ? env.CHANGE_ID.toInteger() : 0 + newCICD_Payload["env"] = env.cicd_environment + newCICD_Payload["job_name"] = env.JOB_NAME ? env.JOB_NAME.split('/')[0] : "UNKNOWN" + newCICD_Payload["sub_job_name"] = env.JOB_NAME ? env.JOB_NAME.split('/')[1] : "UNKNOWN" + newCICD_Payload["build_number"] = env.BUILD_NUMBER.toInteger() + newCICD_Payload["image_tag"] = tag + newCICD_Payload["build_detailed_error"] = env.error_msg_to_db + switch (env.cicd_environment) { + case 'prd': + case 'int': + cicdBaseUrl = 'http://turbo-turtle.meeshogcp.in' + break + default: + cicdBaseUrl = 'http://turbo-turtle.admin.meeshogcp.in' + } + final String newCICD_JSON = writeJSON returnText: true, json: newCICD_Payload + //log.info("New CICD JSON - ${newCICD_JSON}") + final String newCICD_URL = cicdBaseUrl + "/api/v1/ci/jenkins/callback" + final String newCICD_Header = "Content-Type: application/json" + // Use a temporary file to store the JSON payload + // This avoids shell quoting issues completely + final String jsonFilePath = "cicd_payload_${env.BUILD_NUMBER}_${System.currentTimeMillis()}.json" + + try { + // 1. Write the JSON payload to a temporary file + // The writeJSON step ensures the content is valid JSON, escaping internal characters + writeFile(file: jsonFilePath, text: writeJSON(returnText: true, json: newCICD_Payload)) + final String newCICD_JSON_log = readFile(file: jsonFilePath) + log.info("New CICD JSON (from file) - ${newCICD_JSON_log.take(500)}...") // Log a snippet + final def(String response, String code) = sh( + returnStdout: true, + script: """ + curl -s -X POST \\ + -H '$newCICD_Header' \\ + -w '\\n%{response_code}' \\ + $newCICD_URL \\ + -d @$jsonFilePath + """ + ).trim().tokenize("\n") + + if (code != "200") { + log.error("CICD Application API call failed with error code - ${code}, response - ${response}") + } + } catch (Exception e) { + log.error("CICD Application API call failed - " + "Error: " + e.toString()) + throw e + } finally { + sh(script: "rm -f ${jsonFilePath}", returnStatus: true) + } + } + +} + +def callApi(String url, String header, String jsonData){ + try{ + withCredentials([usernamePassword(credentialsId: "ringmaster-token", usernameVariable:'user', passwordVariable: 'token')]){ + final def(String response, String code) = sh(returnStdout: true, script: "curl -s -X POST -H '$header' -H 'Authorization: $token' -w '\\n%{response_code}' $url -d '$jsonData'").trim().tokenize("\n") + log.info("HTTP response status code : ${code}") + if(code != "200"){ + log.error("API call failed with error code - ${code}, response - ${response}") + currentBuild.result = 'FAILURE' + } + } + } + catch ( Exception e) { + log.error("API call failed") + currentBuild.result = 'FAILURE' + throw e + } +} + +@NonCPS +def getDateTime() { + // Get the current date and time in IST + def currentDateTime = ZonedDateTime.now() + // Create a formatter for the desired pattern + def formatter = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd'T'HH:mm:ss") + .appendOffset("+HH:mm", "+00:00") + .toFormatter() + // Format the current date and time using the formatter + def formattedDateTime = currentDateTime.format(formatter) + return formattedDateTime; +} + diff --git a/src/com/meesho/stages/helmGenerator.groovy b/src/com/meesho/stages/helmGenerator.groovy new file mode 100644 index 0000000..5a95545 --- /dev/null +++ b/src/com/meesho/stages/helmGenerator.groovy @@ -0,0 +1,185 @@ +package com.meesho.stages + +import com.meesho.utilities.constructTemplate +import com.meesho.utilities.gitActions +import com.meesho.utilities.buTeamMapping +import com.meesho.utilities.getDockerParams +import com.meesho.stages.multiBranchPipeline + +def run(Map params) { + def btObj = new buTeamMapping() + def gitObj = new gitActions() + + def environments = ['stg':'develop', 'int':'pre-prod', 'prd':'main', 'ftr':'feature'] + final Map<?, ?> modifiedParams = new HashMap<>(params) + + // def helm_repo_name = 'devops-helm-charts' + // def argoRepo = "devops-argo-config" + + def deployment_order = modifiedParams.app_names.replaceAll('\\s', '').split(', ') as List + + modifiedParams['deployment_order'] = deployment_order + + stage('Create Jenkinsfile') { + jenkinsfileCreate(modifiedParams) + } + + modifiedParams['bu'] = btObj.get_bu_initials(modifiedParams['bu']) + modifiedParams['team'] = btObj.get_team_initials(modifiedParams['team']) + log.info(modifiedParams) + + stage('Update Helm charts') { + gitObj.clone("${WORKSPACE}", "${env.helm_repo_name}", null) + environments.each { + modifiedParams['environment'] = it.key + def helm_branch_name = it.value + for (deployment in deployment_order) { + modifiedParams['app_name'] = deployment + modifiedParams['host'] = helm_branch_name == 'feature' ? "INGRESS_PR_NUMBER-${deployment}.dev.internal.meeshotest.in" : "${deployment}.${it.key}.internal.meeshotest.in" + modifiedParams['host'] = (helm_branch_name == 'pre-prod') ? "${deployment}.${modifiedParams.bu}.internal.meesho.co" : modifiedParams.host + //If branch is main, then assuming the env as prod + //Making nodeSelector changes to hypercore or hypermem & based on arch, so that it moves to common node pool + if (helm_branch_name == 'main') { + def cpu = modifiedParams['cpu_request'] + def memory = modifiedParams['memory_request'] + def nodeSelector = 'hypercore' + + if (cpu.contains('m')) { + def numericValue = cpu.replaceAll('\\D+', '').toDouble() + def gb = numericValue / 1000 + cpu = gb.toString() + } + def mem_dgt = memory.replaceAll('\\D+', '').toDouble() + memory = memory.contains('Mi') ? mem_dgt / 1024 : mem_dgt + def ratio = cpu.toDouble() / memory.toDouble() + def arch = modifiedParams['arch'] + def graviton_required = false + if (arch == 'arm64') { + graviton_required = true + } + if (ratio <= 1 / 3) { + if (graviton_required) { + nodeSelector = 'hypermem-arm64' + } else { + nodeSelector = 'hypermem' + } + } else { + if (graviton_required) { + nodeSelector = 'hypercore-arm64' + } else { + nodeSelector = 'hypercore' + } + } + modifiedParams['nodeSelector'] = nodeSelector + } + updateHelmRepo(modifiedParams, env.helm_repo_name, helm_branch_name) + } + } + } + + // stage("Update Argo Application"){ + // gitObj.clone("${WORKSPACE}", "${argoRepo}", null) + // environments.each{ + // modifiedParams["environment"] = it.key + // def argo_branch_name = it.value + // for (deployment in deployment_order){ + // modifiedParams["app_name"] = deployment + // updateArgoRepo(modifiedParams,argoRepo,argo_branch_name) + // } + // } + // } + + stage('Create ECR repo') { + def buildDockerObj = new getDockerParams() + def modules = buildDockerObj.getModules(modifiedParams['repo_name']) + if (modules == null) { + modules = ['module_less'] + } + for (module in modules) { + def ecr_repo_name = (module == 'module_less') ? "${modifiedParams.team}/${modifiedParams.repo_name.toLowerCase()}" : "${modifiedParams.team}/${modifiedParams.repo_name.toLowerCase()}/${module}" + try { + sh "aws ecr create-repository --repository-name ${ecr_repo_name}" + } + catch (Exceptione) { + log.info('repository already present') + } + } + } + stage('Create Multibranch pipeline') { + def multiBranchPipelineObj = new multiBranchPipeline() + multiBranchPipelineObj.applicationOnboard(params['bu'], params['repo_name']) + } +} + +def jenkinsfileCreate(Map config) { + def constructObj = new constructTemplate() + def gitObj = new gitActions() + def repo_name = config.repo_name + def deployment_order = config.deployment_order + def repo_branch_name = 'eks_onboarding' + config['branch_params'] = '' + config['excludedMoudles'] = config.excludedMoudles.replaceAll('\\s', '').split(', ') as List + gitObj.clone("${WORKSPACE}", "${config.repo_name}", null) + gitObj.branchCheckOut(config.repo_name, repo_branch_name) + deployment_yaml_file = get_deployment_yaml_file(config['dockerBuildVersion']) + dir(repo_name) { + sh(script:'mkdir -p deployments/') + sh 'chmod -R 777 .' + constructObj.renderTemplate(config, 'Jenkinsfile', 'Jenkinsfile') + gitObj.add('.', 'Jenkinsfile') + + constructObj.renderTemplate(config, 'config.yaml', 'config.yaml') + gitObj.add('.', 'config.yaml') + + dir('deployments') { + for (deployment in deployment_order) { + config['app_name'] = deployment + constructObj.renderTemplate(config, deployment_yaml_file, deployment + '.yaml') + gitObj.add('.', deployment + '.yaml') + } + } + } + + gitObj.codeCommit(repo_name, repo_branch_name, 'Generating build and deployments files') + gitObj.codePush(repo_name, repo_branch_name) +} + +def updateHelmRepo(Map config, String helm_repo_name, String helm_branch_name) { + def constructObj = new constructTemplate() + def gitObj = new gitActions() + def filepath = "${env.helmChartsPath}/${config.bu}/${config.team}/${config.app_name}" + def file_name = 'values_properties.yaml' + def app_branch = config.app_name + '-' + helm_branch_name + '-helm' + def pr_num = '' + def commit_status = 0 + + gitObj.branchCheckOut(env.helm_repo_name, helm_branch_name) + gitObj.branchCheckOut("${env.helm_repo_name}", "${app_branch}") + // create filepath + dir(env.helm_repo_name) { + sh(script:"mkdir -p ${filepath}" ,returnStdout:true) + sh 'chmod -R 777 .' + constructObj.renderTemplate(config, file_name, filepath + '/' + file_name) + } + + // git commit and push + gitObj.add(env.helm_repo_name, filepath + '/' + file_name) + commit_status = gitObj.codeCommit(env.helm_repo_name, app_branch, 'Generating properties values yaml file') + if (commit_status == 0) { + gitObj.codePush(env.helm_repo_name, app_branch) + pr_num = gitObj.createPR(config.app_name, env.helm_repo_name, helm_branch_name, app_branch, 'Merge helm values properties') + gitObj.mergePR(env.helm_repo_name, pr_num, app_branch) + gitObj.deleteBranch(env.helm_repo_name, helm_branch_name, app_branch) + } +} + +def get_deployment_yaml_file(def dockerBuildVersion) { + switch(dockerBuildVersion) { + case ~/^maven-.*/: return 'deployment.yaml' + case ~/^node-.*/: return 'node-deployment.yaml' + case ~/^python-.*/: return 'python-deployment.yaml' + case ~/^go.*/: return 'go-deployment.yaml' + case 'gradle': return 'gradle-deployment.yaml' + case 'php': return 'php-deployment.yaml' + } +} diff --git a/src/com/meesho/stages/hotFix.groovy b/src/com/meesho/stages/hotFix.groovy new file mode 100644 index 0000000..c74f5b9 --- /dev/null +++ b/src/com/meesho/stages/hotFix.groovy @@ -0,0 +1,24 @@ +package com.meesho.stages + +def run(String repo_name){ + stage(stageName("Check for Hot Fix")){ + dir(repo_name){ + if (env.CHANGE_ID) { + def source_branch_name = env.CHANGE_BRANCH + log.info("Source-Branch : ${source_branch_name}") + source_branch_name = source_branch_name.toLowerCase() + if (source_branch_name.contains("hotfix_")){ + env.hot_fix = true + log.info("***************** Enabling Hot-fix workflow *****************") + } + else if (source_branch_name.matches('^revert-\\d+-.+$')){ + env.hot_fix = true + log.info("***************** Enabling Hot-fix workflow for revert branch *****************") + } + } + else { + log.info("**** Branch: ${env.BRANCH_NAME}, So Skipping Hot-fix check ****") + } + } + } +} \ No newline at end of file diff --git a/src/com/meesho/stages/multiBranchPipeline.groovy b/src/com/meesho/stages/multiBranchPipeline.groovy new file mode 100644 index 0000000..fa6beb1 --- /dev/null +++ b/src/com/meesho/stages/multiBranchPipeline.groovy @@ -0,0 +1,73 @@ +package com.meesho.stages + +import jxl.* +import hudson.util.PersistedList +import jenkins.model.Jenkins +import jenkins.branch.* +import jenkins.plugins.git.* +import org.jenkinsci.plugins.workflow.multibranch.* + +import com.cloudbees.hudson.plugins.folder.* +import org.jenkinsci.plugins.github_branch_source.* +import org.jenkinsci.plugins.workflow.libs.* +import hudson.scm.SCM +import hudson.plugins.git.* +import net.gleske.scmfilter.impl.trait.* + +def applicationOnboard(String foldername, String repo_name) { + // Bring some values in from ansible using the jenkins_script modules wierd "args" approach (these are not gstrings) + String folderName = "${foldername}" + String repoName = "${repo_name}" + String scriptPath = "Jenkinsfile" + String gitRepo = "https://github.com/Meesho/${repo_name}.git" + String mBPName = "${repo_name}-cicd" + String credentialsId = env.GITHUB_CRED + + Jenkins jenkins = Jenkins.instance // saves some typing + + // Get the folder where this job should be + // def folder = jenkins.getItem(folderName) + // //Create the folder if it doesn't exist + // if (folder == null) { + // folder = jenkins.createProject(Folder.class, folderName) + // } + + // Multibranch creation/update + WorkflowMultiBranchProject mbp + def view = jenkins.getView(foldername) + Item item = jenkins.getItem(mBPName) + if ( item != null ) { + // Update case + mbp = (WorkflowMultiBranchProject) item + } else { + // Create case + mbp = jenkins.createProject(WorkflowMultiBranchProject.class, mBPName) + } + + // Configure the script this MBP uses + mbp.getProjectFactory().setScriptPath(scriptPath) + + def implicit = false + def defaultVersion = "master" + def traits = [] + + GitHubSCMSource gitHubSCMSource = new GitHubSCMSource("Meesho", repoName, gitRepo, implicit) + gitHubSCMSource.credentialsId = credentialsId + + BranchDiscoveryTrait branchDiscoveryTrait = new BranchDiscoveryTrait(3) + OriginPullRequestDiscoveryTrait pullRequestTrait = new OriginPullRequestDiscoveryTrait(1) + WildcardSCMHeadFilterTrait wildcardSCMHeadFilterTrait = new WildcardSCMHeadFilterTrait('gcp-main*','','','*') + traits.add(branchDiscoveryTrait) + traits.add(pullRequestTrait) + traits.add(wildcardSCMHeadFilterTrait) + gitHubSCMSource.setTraits(traits) + BranchSource branchSource = new BranchSource(gitHubSCMSource) + NoTriggerBranchProperty noTriggerBranchProperty = new NoTriggerBranchProperty() + BranchProperty[] ntbp = [noTriggerBranchProperty] + branchSource.setStrategy(new DefaultBranchPropertyStrategy(ntbp)) + + PersistedList sources = mbp.getSourcesList() + sources.clear() + sources.add(branchSource) + view.add(mbp) +} diff --git a/src/com/meesho/stages/notify.groovy b/src/com/meesho/stages/notify.groovy new file mode 100644 index 0000000..88edf36 --- /dev/null +++ b/src/com/meesho/stages/notify.groovy @@ -0,0 +1,206 @@ +package com.meesho.stages + +import hudson.Util +import com.meesho.utilities.getDockerParams +import com.meesho.stages.deployRingmaster +import com.meesho.utilities.buTeamMapping + +def run(Map config) { + def maintainer = config.maintainer + def skip_notify = config.skip_notify ?: false + def notify_channel = config.notify_channel && config.notify_channel != '' ? config.notify_channel : 'ci-cd-status' + def build_user = currentBuild.rawBuild.getCause(Cause.UserIdCause).getUserId() + def COLOR_MAP = ['SUCCESS':'good', 'FAILURE':'danger'] + def build_duration = Util.getTimeSpanString(System.currentTimeMillis() - currentBuild.startTimeInMillis) + stage(stageName('Sending notification')) { + if (env.INFRA_ENV == 'toolchain') { + log.info("Toolchain environment detected. Sending signal...") + def dkrobj = new getDockerParams() + def commit_id=dkrobj.getCommitid(config.repo_name) + log.info("Toolchain environment detected. Sending signal...") + def btObj = new buTeamMapping() + def team_initial = btObj.get_team_initials(config.team) + def isNodeBuild = (config.dockerBuildVersion?.toString()?.toLowerCase()?.startsWith('node-')) + def image_path="${env.cicd_environment}/${team_initial}/${config.repo_name}".toLowerCase() + if (env.TOOLCHAIN_ENV && isNodeBuild) { + image_path = "${env.cicd_environment}/${env.TOOLCHAIN_ENV}/${team_initial}/${config.repo_name}".toLowerCase() + log.info("Toolchain: Node build detected, using TOOLCHAIN_ENV-aware image_path: ${image_path}") + } + else { + image_path = "${env.cicd_environment}/${team_initial}/${config.repo_name}".toLowerCase() + log.info("Toolchain: using default image_path (no TOOLCHAIN_ENV segment): ${image_path}") + } + def modules_to_notify = config.modules ?: "" + def imageList = [] + if (modules_to_notify != "") { + imageList = modules_to_notify.collect { module -> + def moduleName = (module instanceof Map) ? module.keySet()[0] : module.toString() + "${env.registry}/${image_path}/${moduleName}:${env.TAG}" + } + } else { + imageList = ["${env.registry}/${image_path}:${env.TAG}"] + } + def payload = [ + "repo_name": config.repo_name, + "build_number": env.BUILD_NUMBER, + "build_status": currentBuild.currentResult, + "image_path": image_path, + "image_tag": env.TAG, + "commit_id": commit_id, + "imageList": imageList.join(',') + ] + + def jsonPayload = writeJSON(returnText: true, json: payload) + + try { + sh """ + curl -k --insecure -X POST -H "Content-Type: application/json" \ + -d '${jsonPayload}' \ + http://172.23.72.116:5002/api/v1/deploy/build-callback + """ + log.info("Toolchain signal sent successfully.") + } catch (Exception e) { + log.error("Failed to send Toolchain signal: ${e.toString()}") + } + return + } + if (build_user == "ringmaster-bot" || build_user == "turbo-turtle") { + def ringmaster_obj = new deployRingmaster() + def btObj = new buTeamMapping() + def team = btObj.get_team_initials(config.team) + ringmaster_obj.run(config.repo_name, config.deployment_order, env.TAG, team, config.dockerBuildVersion, notify_channel) + if(env.cicd_environment == "prd"){ + slackSend color:COLOR_MAP[currentBuild.currentResult], channel:notify_channel, message: "[*${currentBuild.currentResult}*]\n Repo Name - ${config.repo_name}\n maintainer - @${maintainer}\n Time Taken - ${build_duration}\n Build logs - <${BUILD_URL}/console|Open URL>\n TAG for CD - ```${env.TAG}``` Job Message - ```${env.msg}```\n Deploy Applications - <https://ringmaster.meeshogcp.in/applications/cicd/repo/${config.repo_name}/builds|deploy URL> " + } + skip_notify = true + } + if (skip_notify.toBoolean()) { + echo "${config.skip_notify} ${skip_notify}" + log.info('Skipping - Slack Notification.') + } + else { + slackSend color:COLOR_MAP[currentBuild.currentResult], channel:notify_channel, message: "[*${currentBuild.currentResult}*]\n Job Name - <${JOB_URL}|${JOB_NAME}>\n Build Number - <${BUILD_URL}|#${BUILD_NUMBER}>\n Started by - @${maintainer}\n Time Taken - ${build_duration}\n Console Output - <${BUILD_URL}/console|Open URL>\n TAG for CD - ```${env.TAG}``` Job Message - ```${env.msg}```" + } + + if (env.BRANCH_NAME ==~ /(gcp-main|main|gcp-master|master|farmiso-main)/) { + postTrackingApi(config) + if (env.SERVICES != null && (build_user != "ringmaster-bot" || build_user != "turbo-turtle")) { + log.info('Updating Ringmaster deployment history db') + postTrackingRingmasterApi(config) + } + } + else { + log.info('Skipping - Updating in Deployment Tracker.') + } + } +} + +def postTrackingRingmasterApi(Map config){ + log.info('Updating Deployment Details in Ringmaster DB') + final String baseUrl + switch (env.cicd_environment) { + case 'prd': + case 'int': + baseUrl = 'https://ringmaster-api.admin.meeshogcp.in/api/v1/key/update/deployment-history' + break + default: + baseUrl = 'https://ringmaster-api.admin.meeshogcp.in/api/v1/key/update/deployment-history' + } + def working_env = "gcp_${env.cicd_environment}" + final String url = "${baseUrl}?workingEnv=${working_env}" + final String jsonData = getRingmasterJsonData(config) + callRingmasterApi(url, jsonData) +} + +def callRingmasterApi(String url, String jsonData) { + try { + withCredentials([usernamePassword(credentialsId: "ringmaster-token", usernameVariable: 'user', passwordVariable: 'token')]) { + def response = sh( + script: """ + curl -s -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: $token" \ + -d '${jsonData}' \ + ${url} + """, + returnStdout: true + ).trim() + + log.info("API call response: ${response}") + } + } catch (Exception e) { + log.error("API call failed: ${e.message}") + } +} + +def getRingmasterJsonData(Map config) { + def dkrobj = new getDockerParams() + def jsonMap = [:] + + jsonMap['repository'] = config.repo_name + jsonMap['team'] = config.team + jsonMap['link'] = env.BUILD_URL + jsonMap['cd_job'] = env.JOB_NAME + jsonMap['cd_id'] = env.BUILD_NUMBER + jsonMap['initiator'] = currentBuild.rawBuild.getCause(Cause.UserIdCause)?.getUserId() + jsonMap['tag'] = env.TAG + jsonMap['commit_id'] = dkrobj.getCommitid(config.repo_name) + jsonMap['services'] = env.SERVICES ? env.SERVICES.tokenize(',[] ') : [] + jsonMap['status'] = currentBuild.currentResult == 'FAILURE' ? 'FAIL' : currentBuild.currentResult + jsonMap['error_msg'] = env.error_msg_to_db + jsonMap['start_time'] = env.deploymentStartTime + jsonMap['end_time'] = env.deploymentEndTime + jsonMap['working_env'] = "gcp_${env.cicd_environment}" + + def jsonString = writeJSON(returnText: true, json: jsonMap) + return jsonString +} + +def postTrackingApi(Map config) { + final String host = '' + if (env.CLOUD_PROVIDER == 'AWS') { + host = 'http://deployment-tracker.meeshoint.in' + } + else if (env.CLOUD_PROVIDER == 'GCP') { + host = 'http://deployment-tracker.prd.meesho.int' + } + final String path = '/api/1.0/deployment-tracker/jenkins/create' + final String url = "${host}${path}" + final String header = 'Content-Type: application/json' + final String jsonData = getJsonData(config) + + try { + echo "Deployment Tracker URL - ${url}" + final def(String response, String code) = sh(returnStdout: true, script: "curl -s -X POST -H '$header' -w '\\n%{response_code}' $url -d '$jsonData'").trim().tokenize('\n') + log.info("HTTP response status code : ${code}") + if (code != '200') { + log.error("API call failed with error code - ${code}") + } + } + catch ( Exception e) { + log.error('API call failed') + } +} + +def getJsonData(Map config) { + def dkrobj = new getDockerParams() + def build_status_map = ["FAILURE":"FAIL", "SUCCESS":"SUCCESS", "UNSTABLE":"SUCCESS"] + def jsonMap = [:] + jsonMap['repository'] = config.repo_name + jsonMap['team'] = config.team + jsonMap['link'] = env.BUILD_URL + jsonMap['cd_job'] = env.JOB_NAME + jsonMap['cd_id'] = env.BUILD_NUMBER + jsonMap['initiator'] = currentBuild.rawBuild.getCause(Cause.UserIdCause).getUserId() + jsonMap['tag'] = env.TAG + jsonMap['commit_id'] = dkrobj.getCommitid(config.repo_name) + jsonMap['services'] = env.SERVICES ? env.SERVICES.tokenize(',[] ') : null + jsonMap['status'] = currentBuild.currentResult == env.FAILURE ? 'FAIL' : currentBuild.currentResult + jsonMap['error_msg'] = env.error_msg_to_db + + def jsonString = writeJSON returnText: true, json: jsonMap + return jsonString +} + + + diff --git a/src/com/meesho/stages/securityScan.groovy b/src/com/meesho/stages/securityScan.groovy new file mode 100644 index 0000000..b29fe8e --- /dev/null +++ b/src/com/meesho/stages/securityScan.groovy @@ -0,0 +1,24 @@ +package com.meesho.stages + +def run(Map config) { + try { + def repo_name = config.repo_name + stage('Security scan') { + if (config.skip_security_scan) { + log.info('Skipping - Security Scan') + } + else { + final String url = '172.31.5.29:63232/scans' + final def(String response, String code) = sh(returnStdout: true, script: "curl -s -X POST -H 'Content-Type: application/json' -w '\\n%{response_code}' $url -d '{\"reponame\":\"$repo_name\",\"branch\":\"$BRANCH_NAME\"}'").trim().tokenize('\n') + log.info("HTTP response status code : ${code}") + log.info("Response: ${response}") + } + } + } + catch ( Exception e) { + env.msg = 'failed in security scan . Please check console output for more details.' + log.error(env.msg) + currentBuild.result = env.FAILURE + throw e + } +} diff --git a/src/com/meesho/stages/validateBuTeam.groovy b/src/com/meesho/stages/validateBuTeam.groovy new file mode 100644 index 0000000..6aa34d0 --- /dev/null +++ b/src/com/meesho/stages/validateBuTeam.groovy @@ -0,0 +1,18 @@ +package com.meesho.stages + +import com.meesho.utilities.buTeamMapping + +def run(String bu, String team,String module){ + stage("Validate BU and Team"){ + def vObj = new buTeamMapping() + if (vObj.validate(bu,team)){ + log.info("Correct team and BU values") + } + else{ + error "Incorrect BU and team values provided" + } + if(module == ""){ + error "Module can't be empty. if there is no module, Please provide the parameter value as module_less" + } + } +} \ No newline at end of file diff --git a/src/com/meesho/utilities/addSSHKey.groovy b/src/com/meesho/utilities/addSSHKey.groovy new file mode 100644 index 0000000..d92da4f --- /dev/null +++ b/src/com/meesho/utilities/addSSHKey.groovy @@ -0,0 +1,28 @@ +package com.meesho.utilities + +def create() { + withCredentials([file(credentialsId: 'ssh-private-key', variable: 'FILE')]) { + sh """ + cat ${FILE} > ./id_github_jenkins + chmod 600 ./id_github_jenkins + + # Ensure .ssh directory has correct permissions + chmod 700 /root/.ssh + + # Fix SSH config file permissions if it exists + if [ -f /root/.ssh/config ]; then + chmod 600 /root/.ssh/config + chown root:root /root/.ssh/config + fi + + # Fix existing SSH private key permissions if it exists + if [ -f /root/.ssh/id_rsa ]; then + chmod 600 /root/.ssh/id_rsa + chown root:root /root/.ssh/id_rsa + fi + + # Fix any other SSH key files that might exist + find /root/.ssh -type f -name "id_*" -exec chmod 600 {} \\; 2>/dev/null || true + """ + } +} diff --git a/src/com/meesho/utilities/buTeamMapping.groovy b/src/com/meesho/utilities/buTeamMapping.groovy new file mode 100644 index 0000000..3eae348 --- /dev/null +++ b/src/com/meesho/utilities/buTeamMapping.groovy @@ -0,0 +1,125 @@ +/* +Purpose: Utility function to return BU and their respective teams +Author: Avinash kumar Lodhi +*/ +package com.meesho.utilities + +def validate(String bu, String team) { + def bu_team_map = ['supply':['supplier-ads', 'supplier-ads-frontend', 'experience', 'fulfilment', 'fulfilment-frontend','financial-services', 'cataloging', 'cataloging-frontend', 'payout', 'payout-frontend', 'supplier-acquisition-activation', 'supplier-service', 'returns', 'supply-shared', 'display-ads', 'offers','transact', 'supplier-live-commerce'], + 'demand':['comms-platform', 'live-commerce', 'shopping-platform', 'product-feed', 'search', 'product-meta', 'user-growth', 'web', 'transact', 'communications', 'discovery-platform', 'offers', 'android-platform', 'ios', 'demand-shared', 'discovery-ranking'], + 'farmiso':['farmiso'], + 'admin':['devops'], + 'central':['shared', 'devops', 'psec', 'dbe'], + 'dataengg':['data-platform', 'dataengg-shared', 'data-intelligence', 'data-platform-consumption', 'data-platform-ingestion', 'data-platform-nrt', 'data-platform-prism-frmw', 'data-platform-experimentation'], + 'datascience':['data-science', 'ml-platform', 'for-you', 'recommendation', 'catalog-listing-page', 'search', 'advertisement', 'explore', 'pricing', 'product-match', 'catalog-taxonomy', 'brand-infringment', 'fds', 'return-reimbursements', 'fullfilment', 'ugc-moderation-analysis', 'home-page', 'core', 'usergrowth', 'demand-forecast', 'catalog-qc'], + 'mcache':['mcache', 'mcache-shared', 'supplier-service'], + 'infra':['devops', 'dbe'] + ] + if (bu == null || team == null) { + return null + } + return bu_team_map[bu].contains(team) +} + +def get_initials(String targetMap, String targetString) { + def bu_initials = ['supply':'supl', + 'demand':'dmnd', + 'farmiso':'farm', + 'admin':'admn', + 'central':'cntr', + 'dataengg':'deng', + 'datascience':'dsci', + 'mcache':'mche', + 'infra':'infr' + ] + def team_initials = ['supplier-ads':'ads', + 'comms-platform': 'cplat', + 'supplier-ads-frontend':'fads', + 'experience':'xp', + 'fulfilment':'fnf', + 'fulfilment-frontend':'ffnf', + 'financial-services':'fsvc', + 'cataloging':'ctlng', + 'cataloging-frontend':'fctlg', + 'payout':'pay', + 'payout-frontend':'fpay', + 'supplier-acquisition-activation':'saa', + 'supplier-service':'ssvc', + 'seller-services': 'sis', + 'live-commerce':'lcom', + 'shopping-platform':'splat', + 'product-feed':'pfeed', + 'search':'srch', + 'product-meta':'pmeta', + 'user-growth':'grwth', + 'web':'web', + 'transact':'trnst', + 'communications':'comms', + 'discovery-platform':'dplat', + 'farmiso':'farm', + 'devops':'devop', + 'offers':'offer', + 'returns':'retrn', + 'android-platform':'andrd', + 'ios':'ios', + 'shared':'xcntr', + 'supply-shared':'xsupl', + 'demand-shared':'xdmnd', + 'data-platform':'dp', + 'data-science':'ds', + 'ml-platform':'ml', + 'dataengg-shared':'xdeng', + 'datascience-shared':'xdsci', + 'data-intelligence':'di', + 'recommendation':'rcmnd', + 'catalog-listing-page':'ctllp', + 'advertisement':'adv', + 'explore':'explr', + 'pricing':'price', + 'product-match':'patch', + 'catalog-taxonomy':'ctltx', + 'brand-infringment':'brndi', + 'fds':'fds', + 'return-reimbursements':'retrr', + 'fullfilment':'flfmt', + 'ugc-moderation-analysis':'umdra', + 'home-page':'hpage', + 'usergrowth':'ugrwt', + 'demand-forecast':'dmndf', + 'catalog-qc':'ctlqc', + 'data-platform-consumption':'dpcon', + 'data-platform-ingestion':'dping', + 'data-platform-nrt':'dpnrt', + 'data-platform-prism-frmw':'dpprf', + 'data-platform-experimentation':'dpexp', + 'display-ads':'dplay', + 'discovery-ranking':'drank', + 'mcache':'mche', + 'mcache-shared':'xmche', + 'supplier-live-commerce':'slcom', + 'trust-and-safety': 'tns', + 'valmo': 'vlm', + 'psec':'psec', + 'dbe':'dbe', + 'dev-productivity':'devprd'] + + if (targetString == null) { + return null + } + switch (targetMap) { + case 'bu_initials': + return bu_initials[targetString] + case 'team_initials': + return team_initials[targetString] + default: + return 'Undefined option' + } +} + +def get_team_initials(String team) { + return get_initials('team_initials', team) +} + +def get_bu_initials(String bu) { + return get_initials('bu_initials', bu) +} diff --git a/src/com/meesho/utilities/constructParam.groovy b/src/com/meesho/utilities/constructParam.groovy new file mode 100644 index 0000000..c467d49 --- /dev/null +++ b/src/com/meesho/utilities/constructParam.groovy @@ -0,0 +1,391 @@ + +package com.meesho.utilities + +def getWhitelistedRepos(fileName){ + dir('whitelist'){ + git( + url: "https://github.com/Meesho/whitelists.git", + branch: "main", + credentialsId: 'cicd-github-app', + )} + def yaml = readYaml file: "whitelist/${fileName}.yaml" + return yaml.get("repos", []) as Set +} + +def getWhitelistedDeployable(fileName, keyName){ + dir('whitelist'){ + git( + url: "https://github.com/Meesho/whitelists.git", + branch: "main", + credentialsId: 'cicd-github-app', + )} + def yaml = readYaml file: "whitelist/${fileName}.yaml" + return yaml.get(keyName, []) as Set +} + +/* +* return `true` if we should not proceed +*/ +def isMultizoneEnabled( String deployable){ + def WHITELIST = getWhitelistedDeployable("multizone-enabled-repos" , "multizone_enabled_deployables") + if (WHITELIST.contains(deployable)) { + return true + } + return false +} + +/* +* return `true` if we should not proceed +*/ +def skipSonarCheckForbidden(Map config, Map environment_map) { + def branch = env.BRANCH_NAME + def environment = environment_map.getOrDefault(branch, "int") + + def WHITELIST = getWhitelistedRepos("skip-sonar-whitelist") + + // returns 'true' if we aren't skipping sonar + // or if we're allowed to skip sonar + def build_version = config['dockerBuildVersion'] + def repo_name = config['repo_name'] + + // if not a maven build OR if it a hotfix we exit early and don’t care what skip_sonar is + if (WHITELIST.contains(repo_name)|| !(build_version.contains("maven")) || environment != "prd" || branch.contains("hotfix")){ + return false; + } + + return config['skip_sonar']; +} + +def skipSonarCheckForGo(Map config) { + def branch = env.BRANCH_NAME + def WHITELIST = getWhitelistedRepos("skip-sonar-whitelist") + def build_version = config['dockerBuildVersion'] + def repo_name = config['repo_name'] + echo "env.INFRA_ENV: ${env.INFRA_ENV}" + if (WHITELIST.contains(repo_name)|| branch.contains("hotfix")|| env.INFRA_ENV == "toolchain"){ + return true; + } + return false; +} +/* +* return `true` if we should not proceed +*/ +def appConfigDisabledForbidden(boolean appConfigEnabled, String repo_name, String environment, String build_version){ + def branch = env.CHANGE_TARGET + def WHITELIST = getWhitelistedRepos("app-config-disabled") + if (WHITELIST.contains(repo_name) || environment!="stg"){ + return false + } + if (!(build_version.contains("maven") || build_version.contains("gradle"))){ + return false + } + return !appConfigEnabled +} + +/* +* return `true` if we should not proceed +*/ +def allowedNonDevelopPrDeploymentToIntRepos( String repo_name){ + def WHITELIST = getWhitelistedRepos("allowedNonDevelopPrDeploymentToInt") + if (WHITELIST.contains(repo_name)) { + return true + } + return false +} + +/* +* return `true` if we should not proceed +*/ +def ValidateCacConfigForRepo(boolean ValidateConfig, String repo_name ){ + def branch = env.CHANGE_TARGET + def WHITELIST = getWhitelistedRepos("ValidateCacConfig") + if (WHITELIST.contains(repo_name)) { + return true + } + return ValidateConfig +} + +def getToolchainEnv() { + def paramsAction = currentBuild.rawBuild.getAction(hudson.model.ParametersAction.class) + if (paramsAction) { + echo "paramsAction: ${paramsAction}" + def p = paramsAction.getParameter("TOOLCHAIN_ENV") + echo "p: ${p}" + if (p) { + return p.getValue()?.toString() + } + } + return null +} + +def run(Map config) { + def branch_name = env.BRANCH_NAME + if (env.INFRA_ENV == 'toolchain' && (config.build_tool?.startsWith('node-') || config.dockerBuildVersion?.startsWith('node-'))) { + branch_name = 'develop' // develop maps to stg in the environment_map + def tcEnv = getToolchainEnv() + env.TOOLCHAIN_ENV = tcEnv + log.info("Successfully extracted TOOLCHAIN_ENV from trigger cause: ${env.TOOLCHAIN_ENV}") + } + def environment_map = ['master':'prd', 'main':'prd', 'develop':'stg', 'gcp-main':'prd', 'farmiso-main':'prd', 'gcp-master':'prd', 'gcp-dev':'stg'] + env.skip_user_input = config.skip_user_input ?: false + + // don't allow skip_sonar + if (skipSonarCheckForbidden(config, environment_map)){ + throw new Exception("Not allowed to skip sonar (skip_sonar in config.yaml)") + } + + if (env.CHANGE_ID) { + branch_name = env.CHANGE_TARGET + environment_map = ['master':'int', 'main':'int', 'gcp-main':'int', 'farmiso-main':'int', 'gcp-master':'int', 'develop':'ftr', 'gcp-dev':'ftr'] + } + environment_map[branch_name] = environment_map[branch_name] ?: 'ftr' + + if (config.containsKey('branch_params')) { + Map branch_config = config['branch_params'].collectEntries { key, value -> branch_name.matches(key) ? value : [ : ] } + config.remove('branch_params') + config.putAll(branch_config) + } + if (config.containsKey('environment')) { + Map envrionment_config = config['environment'].collectEntries { key, value -> environment_map[branch_name].matches(key) ? value : [ : ] } + config.remove('environment') + config.putAll(envrionment_config) + } + + env.GITHUB_CRED = 'svc-devops-meesho' + env.cicd_environment = environment_map[branch_name] + env.helm_repo_name = 'devops-helm-charts' + env.argo_repo_name = 'devops-argo-config' + + echo "Branch Name - ${branch_name} and Environment - ${env.cicd_environment}" + if (env.CLOUD_PROVIDER == 'AWS') { + def prodAccountID = '847438129436' + def prodRegion = 'ap-southeast-1' + def prodObjBucket = 'meesho-prod-artifacts' + def devAccountID = '766380763301' + def devObjBucket = 'meesho-stg-artifacts' + def devRegion = 'ap-south-1' + def accountDetails = [ + 'prd': [ + 'accountID': prodAccountID, + 'region': prodRegion, + 'objBucket': prodObjBucket + ], + 'int': [ + 'accountID': prodAccountID, + 'region': prodRegion, + 'objBucket': prodObjBucket + ], + 'stg': [ + 'accountID': devAccountID, + 'region': devRegion, + 'objBucket': devObjBucket + ], + 'ftr': [ + 'accountID': devAccountID, + 'region': devRegion, + 'objBucket': devObjBucket + ] + ] + env.accountID = accountDetails[env.cicd_environment]['accountID'] + env.region = accountDetails[env.cicd_environment]['region'] + env.registry = "${env.accountID}.dkr.ecr.${env.region}.amazonaws.com" + env.buildRegistry = env.registry + env.helmChartsPath = 'charts' + env.defaultHelmChartVersion = '1.0.10' + env.objBucket = accountDetails[env.cicd_environment]['objBucket'] + env.skip_notify = false + echo "${env.accountID}.dkr.ecr.${env.region}.amazonaws.com" + } + else if (env.CLOUD_PROVIDER == 'GCP') { + def prodVaultURL = 'https://vault-prd.meeshogcp.in' + def prodVaultToken = 'vault-prd-token' + def prodSonarURL = 'https://sonarqube-prd.meeshogcp.in' + def prodSonarToken = 'sonar-token-prod' + def prodSonarEnv = 'sonarqube-test' + def prodGoProxyUrl = 'https://athens-prd.meeshogcp.in' + def prdDockerHost = 'dind-prd-svc' + def preProdDockerHost = 'dind-int-svc' + def prodGCPProject = "meesho-${config.bu}-prd-0622" + def preprodGCPProject = "meesho-shared-int-0525" + def devVaultURL = 'https://vault-dev.meeshogcp.in' + def devVaultToken = 'vault-dev-token' + def devSonarURL = "https://sonarqube-${config.bu}-dev.meeshogcp.in" + def devSonarToken = "sonar-token-${config.bu}-dev" + def devSonarEnv = "sonar-${config.bu}-dev" + def devGoProxyUrl = 'https://athens-dev.meeshogcp.in' + def devGCPProject = "meesho-${config.bu}-dev-0622" + def devDockerHost = 'dind-dev-new-svc.jenkins-new.svc.cluster.local' + def toolchainDockerHost = 'toolchain-dind-dev-svc.jenkins-toolchain.svc.cluster.local' + def accountDetails = [ + 'prd': [ + 'vaultURL': prodVaultURL, + 'vaultToken': prodVaultToken, + 'sonarURL': prodSonarURL, + 'sonarToken': prodSonarToken, + 'GCPProject': prodGCPProject, + 'sonarEnv': prodSonarEnv, + 'GCPLBProject': prodGCPProject, + 'goProxyUrl': prodGoProxyUrl, + 'dockerHost': prdDockerHost + ], + 'int': [ + 'vaultURL': prodVaultURL, + 'vaultToken': prodVaultToken, + 'sonarURL': prodSonarURL, + 'sonarToken': prodSonarToken, + 'GCPProject': preprodGCPProject, + 'sonarEnv': prodSonarEnv, + 'GCPLBProject': prodGCPProject, + 'goProxyUrl': prodGoProxyUrl, + 'dockerHost': preProdDockerHost + ], + 'stg': [ + 'vaultURL': devVaultURL, + 'vaultToken': devVaultToken, + 'sonarURL': devSonarURL, + 'sonarToken': devSonarToken, + 'GCPProject': devGCPProject, + 'sonarEnv': devSonarEnv, + 'GCPLBProject': devGCPProject, + 'goProxyUrl': devGoProxyUrl, + 'dockerHost': devDockerHost + ], + 'ftr': [ + 'vaultURL': devVaultURL, + 'vaultToken': devVaultToken, + 'sonarURL': devSonarURL, + 'sonarToken': devSonarToken, + 'GCPProject': devGCPProject, + 'sonarEnv': devSonarEnv, + 'GCPLBProject': devGCPProject, + 'goProxyUrl': devGoProxyUrl, + 'dockerHost': devDockerHost ] + ] + env.GCPProject = accountDetails[env.cicd_environment]['GCPProject'] + env.GCPLBProject = accountDetails[env.cicd_environment]['GCPLBProject'] + env.registry = 'asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622' + if (env.INFRA_ENV == 'toolchain') { + env.registry = 'asia-southeast1-docker.pkg.dev/meesho-central-dev-0622/toolchain' + } + env.buildRegistry = 'asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/admin' + env.helmChartsPath = env.cicd_environment == 'prd' ? 'values_v3' : 'values_v2' + env.defaultHelmChartVersion = '2.0.0' + env.objBucket = "gcs-infr-dvps-meesho-artifacts-${env.cicd_environment}" + env.vaultURL = accountDetails[env.cicd_environment]['vaultURL'] + env.vaultToken = accountDetails[env.cicd_environment]['vaultToken'] + env.sonarURL = accountDetails[env.cicd_environment]['sonarURL'] + env.sonarToken = accountDetails[env.cicd_environment]['sonarToken'] + env.sonarEnv = accountDetails[env.cicd_environment]['sonarEnv'] + env.goProxyUrl = accountDetails[env.cicd_environment]['goProxyUrl'] + env.skip_notify = true + env.DOCKER_HOST = accountDetails[env.cicd_environment]['dockerHost'] + if (env.INFRA_ENV == 'toolchain') { + env.DOCKER_HOST = toolchainDockerHost + } + } + echo "Bucket and Image Repo Details - ${env.registry} ${env.buildRegistry} ${env.objBucket}" + echo "Docker Host - ${env.DOCKER_HOST}" +} + +def perDeploymentVars(Map value_binding) { + env.BU = value_binding.bu + echo "${env.BU}" + + if (env.CLOUD_PROVIDER == 'AWS') { + def inClusterName = 'https://kubernetes.default.svc' + def prodArgoURL = 'prod-ops-argocd.meesho.com' + def devArgoURL = 'stg-dev-argocd.meeshotest.in' + def prodK8sCluster = [ + 'supply': 'https://211689C65F4496AAA76FE19B29E24B6E.yl4.ap-southeast-1.eks.amazonaws.com', + 'demand': 'https://9059D138B6277A0EA592BA7F4B680CEC.gr7.ap-southeast-1.eks.amazonaws.com', + 'dataengg': 'https://806ADE97231CA65D2A0FFB780352630D.yl4.ap-southeast-1.eks.amazonaws.com', + 'datascience': 'https://E34D119516F751AFD1A61043B0514726.yl4.ap-southeast-1.eks.amazonaws.com', + 'central': 'https://C95FCEDF7CEE890F531E1D0488BEC6C3.gr7.ap-southeast-1.eks.amazonaws.com', + 'mcache': 'https://2FFADF214AD8BE58769C5F2797987E50.gr7.ap-southeast-1.eks.amazonaws.com' + ] + def devK8sCluster = [ + 'supply': inClusterName, + 'demand': inClusterName, + 'dataengg': inClusterName, + 'datascience': inClusterName, + 'central': inClusterName, + 'mcache': inClusterName + ] + def accountDetails = [ + 'prd': [ + 'argoURL': prodArgoURL, + 'argoIncubator': 'prod-app-of-apps', + 'serverMap': prodK8sCluster + ], + 'int': [ + 'argoURL': prodArgoURL, + 'argoIncubator': 'int-app-of-app', + 'serverMap': prodK8sCluster + ], + 'stg': [ + 'argoURL': devArgoURL, + 'argoIncubator': 'app-of-apps', + 'serverMap': devK8sCluster + ], + 'ftr': [ + 'argoURL': devArgoURL, + 'argoIncubator': 'ftr-app-of-apps', + 'serverMap': devK8sCluster + ] + ] + env.clusterName = accountDetails[env.cicd_environment]['serverMap'][env.BU] + env.argoAppsPath = 'applications' + env.argoURL = accountDetails[env.cicd_environment]['argoURL'] + env.argoCreds = 'argocd-jenkins' + env.argoIncubator = accountDetails[env.cicd_environment]['argoIncubator'] + env.argoAppNS = 'argocd' + } + else if (env.CLOUD_PROVIDER == 'GCP') { + def prodArgoURL = "argocd-${env.BU}-prd.meeshogcp.in" + def prodArgoCreds = "argocd-${env.BU}-prd-creds" + def preprodArgoURL = "argocd-shared-int.meeshogcp.in" + def preprodArgoCreds = "argocd-shared-int-creds" + def devArgoURL = 'argocd-dev.meeshogcp.in' + def devArgoCreds = 'argocd-dev-creds' + def accountDetails = [ + 'prd': [ + 'argoURL': prodArgoURL, + 'argoCreds': prodArgoCreds, + 'argoAppNS': "argocd-${env.BU}-prd", + 'clusterName': "k8s-${env.BU}-prd-ase1" + ], + 'int': [ + 'argoURL': preprodArgoURL, + 'argoCreds': preprodArgoCreds, + 'argoAppNS': "argocd-shared-int", + 'clusterName': "k8s-shared-int-ase1" + ], + 'stg': [ + 'argoURL': devArgoURL, + 'argoCreds': devArgoCreds, + 'argoAppNS': "argocd-dev", + 'clusterName': "k8s-${env.BU}-stg-ase1" + ], + 'ftr': [ + 'argoURL': devArgoURL, + 'argoCreds': devArgoCreds, + 'argoAppNS': "argocd-dev", + 'clusterName': "k8s-${env.BU}-stg-ase1" + ] + ] + + env.clusterName = accountDetails[env.cicd_environment]['clusterName'] + + if (env.cicd_environment == 'int') { + env.argoAppsPath = "applications_v2/k8s-${env.BU}-int-ase1" + } else { + env.argoAppsPath = "applications_v2/${env.clusterName}" + } + + env.argoURL = accountDetails[env.cicd_environment]['argoURL'] + env.argoCreds = accountDetails[env.cicd_environment]['argoCreds'] + env.argoAppNS = accountDetails[env.cicd_environment]['argoAppNS'] + env.argoIncubator = "incubator-apps-k8s-${env.BU}-${env.cicd_environment}-ase1" + } + echo "${env.clusterName} ${env.argoURL} ${env.argoIncubator}" +} diff --git a/src/com/meesho/utilities/constructTemplate.groovy b/src/com/meesho/utilities/constructTemplate.groovy new file mode 100644 index 0000000..1a5ec05 --- /dev/null +++ b/src/com/meesho/utilities/constructTemplate.groovy @@ -0,0 +1,51 @@ +package com.meesho.utilities + +def run(Map binding, String text) { + return _construct(binding, text) +} + +def renderTemplate(Map binding, String templateFile, String fileName) { + def template = libraryResource 'com/meesho/' + templateFile + // Generic add_file support (language-agnostic): user repos declare files to bake + // into the image under `add_file` in config.yaml -- either a single {path, target} + // mapping or a list of them. renderTemplate always runs inside dir(repo_name), so + // config.yaml resolves to the repo's config. The Dockerfile templates COPY each entry + // into the image. Set for every render so the `add_files.each { ... }` guard never + // hits a missing binding var. This is the one language-agnostic convention for + // mounting extra files (e.g. the ab-client SDK's configs/abacus_experiments.yml -> + // /opt/config/abacus_experiments.yml). + binding['add_files'] = resolveAddFiles() + def renderedTemplate = run(binding, template.toString()) + writeFile file:fileName, text: renderedTemplate +} + +// resolveAddFiles reads config.yaml (if present) and normalises the optional `add_file` +// list into a list of [path, target] maps. `add_file` is a list of {path, target} +// entries; each entry's target defaults to /opt/target when omitted; entries without a +// path are skipped. A non-list add_file is ignored. +def resolveAddFiles() { + if (!fileExists('config.yaml')) { + return [] + } + def cfg = readYaml file: 'config.yaml' + def entries = cfg?.add_file + if (!(entries instanceof List)) { + return [] + } + def result = [] + entries.each { entry -> + if (entry?.path) { + result.add([path: entry.path, target: entry.target ?: '/opt/target']) + } + } + return result +} + +@NonCPS +def _construct(Map binding, String text) { + binding = new HashMap(binding) + def engine = new groovy.text.SimpleTemplateEngine() + def template = engine.createTemplate(text).make(binding) + engine = null + return template.toString() +} diff --git a/src/com/meesho/utilities/dockerUtilities.groovy b/src/com/meesho/utilities/dockerUtilities.groovy new file mode 100644 index 0000000..05018af --- /dev/null +++ b/src/com/meesho/utilities/dockerUtilities.groovy @@ -0,0 +1,52 @@ +package com.meesho.utilities + +def retryDockerPush(String cmd) { + int maxAttempts = 5 + int attempt = 1 + while (attempt <= maxAttempts) { + try { + sh cmd + break + } catch (err) { + if (attempt == maxAttempts) { + error("Command failed after ${maxAttempts} attempts: ${err}") + } + echo "Command failed, retrying... (${attempt}/${maxAttempts})" + sleep 3 + attempt++ + } + } +} + +def imageExists(String registry, String repoName, String tag) { + int maxAttempts = 5 + int attempt = 1 + + while (attempt <= maxAttempts) { + try { + if (env.CLOUD_PROVIDER == 'GCP') { + def result = sh( + script: "gcloud container images list-tags ${registry}/${repoName} --filter='tags:${tag}' --format='get(tags)'", + returnStdout: true + ).trim() + return result != "" + } + else if (env.CLOUD_PROVIDER == 'AWS') { + echo "AWS not supported." + return false + } + } catch (Exception e) { + echo "Attempt ${attempt}/${maxAttempts} failed: Error checking image existence: ${e.toString()}" + + if (attempt == maxAttempts) { + echo "Max attempts reached. Assuming image does not exist or service is down." + return false + } + + echo "Retrying in 5 seconds..." + sleep 5 + attempt++ + } + } + return false +} \ No newline at end of file diff --git a/src/com/meesho/utilities/getDockerParams.groovy b/src/com/meesho/utilities/getDockerParams.groovy new file mode 100644 index 0000000..3893640 --- /dev/null +++ b/src/com/meesho/utilities/getDockerParams.groovy @@ -0,0 +1,61 @@ +package com.meesho.utilities + +def getCommitid(String repo_name) { + dir(repo_name) { + def gitCmd = env.INFRA_ENV == 'toolchain' ? 'git -c safe.directory="$(pwd)"' : 'git' + def commitID = sh(returnStdout: true, script: "${gitCmd} log -1 --format=%h").trim() + env.commit_id = sh(returnStdout: true, script: "${gitCmd} log -1 --format=%H").trim() + return commitID + } +} + +def getVersion(String repo_name) { + dir(repo_name) { + if (fileExists('pom.xml')) { + return sh(returnStdout: true, script: 'xq -r .project.version pom.xml').trim() + } + else if (fileExists('package.json')) { + return sh(returnStdout: true, script: 'jq -r .version package.json').trim() + } + else { + return '1.0' + } + } +} + +def getModules(String repo_name) { + dir(repo_name) { + if (fileExists('pom.xml')) { + modules = sh(returnStdout: true, script: 'xq -r .project.modules.module[] pom.xml 2>/dev/null || xq -r .project.modules.module pom.xml 2>/dev/null || echo empty').trim() + if (modules == 'empty' || modules == 'null') { + return null + } + modules = modules.split('\n') as List + return modules + } + else { + return null + } + } +} + +def getTag(String repo_name) { + def version = getVersion(repo_name) + def commitID = getCommitid(repo_name) + def date = new Date() + def timesha = date.getTime() + def tag = "v${version}-${commitID}-${timesha}" + if (env.INFRA_ENV == 'toolchain') { + tag = "v${version}-${commitID}" + } + + return tag +} + +def getTagShort(String repo_name) { + def version = getVersion(repo_name) + def commitID = getCommitid(repo_name) + def tagShort = "v${version}-${commitID}" + + return tagShort +} diff --git a/src/com/meesho/utilities/getYamlParameter.groovy b/src/com/meesho/utilities/getYamlParameter.groovy new file mode 100644 index 0000000..eb116cf --- /dev/null +++ b/src/com/meesho/utilities/getYamlParameter.groovy @@ -0,0 +1,16 @@ +package com.meesho.utilities + +def getParam(String wd, String fileName = 'config.yaml') { + dir(wd) { + def config = readYaml file: fileName + return config + } +} +//read as string +def getParamAsString(String wd, String fileName = 'config.yaml') { + dir(wd) { + // Read the entire file content as a string + def yamlContent = readFile(file: fileName) + return yamlContent + } +} diff --git a/src/com/meesho/utilities/gitActions.groovy b/src/com/meesho/utilities/gitActions.groovy new file mode 100644 index 0000000..014e641 --- /dev/null +++ b/src/com/meesho/utilities/gitActions.groovy @@ -0,0 +1,342 @@ +package com.meesho.utilities + +def clone(String path, String repo_name, String branch_name) { + log.info("Cloning repo - ${repo_name}, branch - ${branch_name} in path - ${path}") + dir(path) { + try { + sh "rm -rf ${repo_name}" + } + catch (Exception e) { + log.info('Repo not present, proceeding to clone the repo.') + } + try { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + if (branch_name) { + sh "git clone -b ${branch_name} https://github.com/Meesho/${repo_name}.git" + } + else { + sh "git clone https://github.com/Meesho/${repo_name}.git" + } + } + } + catch (Exception e) { + env.msg += "\n\nFAILED -\n ```Unable to clone repo - ${repo_name}, branch - ${branch_name} in path - ${path}.\n Full Erroror Details - ${e}```" + env.error_msg_to_db += "Unable to Clone Repo ${repo_name};" + log.error("${env.msg}") + throw e + } + } +} + +def branchCheckOut(String path, String branch_name) { + log.info("Checking out to branch - ${branch_name} in path - ${path}") + dir(path) { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh 'git fetch' + try { + sh "git checkout ${branch_name}" + sh 'git pull --ff-only' + } + catch (Exception e) { + log.info('Creating new branch') + sh "git checkout -b ${branch_name}" + } + sh 'git branch' + } + } +} + +def add(String path, String git_add_file) { + log.info("Adding files - ${git_add_file} to git, in path - ${path}") + try { + dir(path) { + sh "git add ${git_add_file}" + } + } + catch (Exception e) { + env.msg += "\n\nFAILED -\n ```Unable to add files - ${git_add_file} to git, in path - ${path}.\n Full Erroror Details - ${e}```" + env.error_msg_to_db += 'Unable to Add Files;' + log.error("${env.msg}") + throw e + } +} + +def createTag(path, git_tag, tag_message) { + log.info("Creating tag - ${git_tag} in git, in path - ${path}") + try { + dir(path) { + sh "git tag ${git_tag} -m '${tag_message}'" + } + } + catch (Exception e) { + env.msg += "\n\nFAILED -\n ```Unable to create tag - ${git_tag} in git, in path - ${path}.\n Full Erroror Details - ${e}```" + env.error_msg_to_db += "Unable to Create Tag ${git_tag};" + log.error("${env.msg}") + throw e + } +} + +def codeCommit(String path, String branch_name, String commit_message) { + log.info("Commiting code with message - ${commit_message} in git, in path - ${path}") + def global_arg = '--global' + def gitName = 'svc-devops-meesho' + def gitEmail = 'devops@meesho.com' + def gitEnv = [ + "GIT_AUTHOR_NAME=${gitName}", + "GIT_AUTHOR_EMAIL=${gitEmail}", + "GIT_COMMITTER_NAME=${gitName}", + "GIT_COMMITTER_EMAIL=${gitEmail}", + ] + dir(path) { + withEnv(gitEnv) { + if (sh(returnStatus: true, script: "git diff-index --quiet ${branch_name} 2>/dev/null")) { + try { + sh "git commit -m '${commit_message}'" + return 0 + } + catch (Exception e) { + env.msg += "\n\nFAILED -\n ```Unable to commit code with message - ${commit_message} to git in path - ${path}.\n Full Erroror Details - ${e}```" + env.error_msg_to_db += 'Unable to Commit;' + log.error("${env.msg}") + return 1 + } + } + else { + log.info("No changes were made in branch - ${branch_name}") + return 1 + // log.error("${env.msg}") + } + } + } +} + +def codePush(String path, String branch_name) { + log.info("Pushing code in branch - ${branch_name} in path - ${path}") + try { + dir(path) { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + try { + sh "git push -f --set-upstream origin ${branch_name}" + } + catch (Exception e) { + sh 'git pull --ff-only' + sh 'git push -f' + } + } + } + } + catch (Exception e) { + env.msg += "\n\nFAILED -\n ```Unable to push code in branch - ${branch_name} in path - ${path}.\n Full Erroror Details - ${e}```" + env.error_msg_to_db += "Unable to push code in branch - ${branch_name};" + log.error("${env.msg}") + throw e + } +} + +def tagPush(String path) { + log.info("Pushing tag in path - ${path}") + try { + dir(path) { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh 'git push --tags' + } + } + } + catch (Exception e) { + env.msg += "\n\nFAILED -\n ```Unable to push tag from path - ${path}.\n Full Erroror Details - ${e}```" + env.error_msg_to_db += 'Unable to Push tag;' + log.error("${env.msg}") + throw e + } +} + +def createPR(String app_name, String repo_name, String base_branch, String target_branch, String pr_message) { + log.info("Creating PR with message - ${pr_message} in Github for repo ${repo_name}") + def body = "{\"title\":\"${app_name} ${base_branch} Onboarding\",\"body\":\"${pr_message}\",\"head\":\"${target_branch}\",\"base\":\"${base_branch}\"}" + def pr_num = 'empty' + + try { + withCredentials([usernamePassword(credentialsId: "${env.GITHUB_CRED}", usernameVariable:'user', passwordVariable: 'token')]) { + create_pr = httpRequest httpMode: 'POST', + customHeaders: [ + [name: 'Accept', value: 'application/vnd.github+json'], + [maskValue: true, name: 'Authorization', value: 'Bearer ' + token] + ], + requestBody: body, + url: "https://api.github.com/repos/Meesho/${repo_name}/pulls", + validResponseCodes: '201', + timeout: 10 + def create_pr_json = readJSON(text: create_pr.content) + try { + error_filter = create_pr_json.errors.message[0] + } + catch (Exception e) { + error_filter = 'empty' + } + if ( error_filter.contains('No commits between') ) { + log.info("No changes were made in branch - ${target_branch}. Skipping - PR Creation...\n${create_pr_json}") + } + else if ( error_filter.contains('A pull request already exists') ) { + log.info(error_filter) + } + else { + pr_num = create_pr_json.number + pr_url = create_pr_json.url + log.info("PR Number - ${pr_num} and URL - ${pr_url}") + } + return pr_num.toString() + } + } + catch (Exception e) { + env.msg = "\n\nFAILED -\n Unable to create PR with message - ${pr_message}.\n Full Error Details - ${e}" + def error_code = "${e}".split('Status code')[1] + error_code = error_code.toString() + error_code = error_code.split(' ') + error_code = error_code[1] + env.error_part_msg_to_db = "Unable to Create PR for base branch - ${base_branch}, target branch - ${target_branch}, Repo Name - ${repo_name}. Failed with status code ${error_code};" //Akshay has asked to remove it + log.error("${env.msg}") + throw e + } +} + +def mergePR(String repo_name, String pr_num, String target_branch) { + log.info("Merging PR ${pr_num} in repo - ${repo_name}") + if ( "${pr_num}" == 'empty' || pr_num == null) { + log.info("No changes were made in branch - ${target_branch}. Skipping - PR Merge...") + } + else { + try { + withCredentials([usernamePassword(credentialsId: "${env.GITHUB_CRED}", usernameVariable:'user', passwordVariable: 'token')]) { + merge_pr = httpRequest httpMode: 'PUT', + customHeaders: [ + [name: 'Accept', value: 'application/vnd.github+json'], + [maskValue: true, name: 'Authorization', value: 'Bearer ' + token] + ], + url: "https://api.github.com/repos/Meesho/${repo_name}/pulls/${pr_num}/merge", + validResponseCodes: '200', + timeout: 10 + + def merge_pr_json = readJSON(text: merge_pr.content) + if ( merge_pr_json.message.contains('not mergeable') ) { + env.msg += "\n\nFAILED -\n Unable to merge PR - ${pr_num}. PR URL - https://github.com/Meesho/${repo_name}/pulls/${pr_num}. ${merge_pr_json.message}" + log.error("${env.msg}") + error("${env.msg}") + } + else { + log.info("PR Merge was SUCCESSFUL. Message - ${merge_pr_json.message}") + } + } + } + catch (Exception e) { + env.msg = "\n\nFAILED -\n Unable to merge PR - ${pr_num} \n Full Error Details - ${e}" + def error_code = "${e}".split('Status code')[1] + error_code = error_code.toString() + error_code = error_code.split(' ') + error_code = error_code[1] + env.error_part_msg_to_db = "Unable to Merge PR - ${pr_num}, Repo Name - ${repo_name} status code ${error_code};" //Akshay has asked to remove it + log.error("${env.msg}") + throw e + } + } +} + +def deleteBranch(String path, String base_branch, String target_branch) { + log.info("Deleting branch - ${target_branch}") + dir(path) { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh 'git fetch' + try { + sh "git checkout ${base_branch}" + sh "git branch -d ${target_branch}" + sh "git push -d origin ${target_branch}" + } + catch (Exception e) { + env.msg = "\n\nFAILED -\n Unable to delete branch - ${target_branch}.\n Full Error Details - ${e}" + env.error_msg_to_db = "Unable to Delete Branch ${target_branch};" + log.error("${env.msg}") + throw e + } + sh 'git branch' + } + } +} + +def preDeleteBranch(String path, String base_branch, String target_branch) { + log.info("Pre Deleting branch - ${target_branch}") + dir(path) { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + try { + sh 'git fetch' + sh "git checkout ${base_branch}" + sh "git push -d origin ${target_branch}" + } + catch (Exception e) { + echo "Pre Delete Branch - Unable to delete branch ${target_branch}" + } + sh 'git branch' + } + } +} + +def fetchDiffFilesForPullRequest(String path , String target_branch){ + log.info("check diff from target branch - ${target_branch}") + dir(path) { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + try { + sh "git fetch origin ${target_branch}:${target_branch}" + sh "git branch" + def changedFiles = sh(script: "git diff --name-only HEAD ${target_branch}", returnStdout: true).trim() + echo "${changedFiles}" + return changedFiles + } + catch (Exception e) { + echo "failed to get the diff files from ${target_branch}" + } + } + } +} +def fetchDiffFilesForPushRequest(String path ){ + log.info("check diff of current and previous commit for ${path}") + dir(path) { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + try { + sh "git fetch origin" + + // Get the latest commit hash and the previous commit hash + def currentCommit = sh(script: "git rev-parse HEAD", returnStdout: true).trim() + def previousCommit = sh(script: "git rev-parse HEAD~1", returnStdout: true).trim() + + // Capture the list of changed files in a Groovy variable + def changedFiles = sh(script: "git diff --name-only ${previousCommit} ${currentCommit}", returnStdout: true).trim() + + return changedFiles + } + catch (Exception e) { + echo "failed to get the diff files for ${path}" + } + } + } +} + +def fetchLatestCommitId(String path, String branch) { + log.info("Checking the latest commit for branch '${branch}' in path '${path}'") + dir(path) { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + try { + // Fetch the latest changes for the specific branch + sh "git fetch origin ${branch}:${branch}" + + // Check out the specified branch + sh "git checkout ${branch}" + + // Get the latest commit hash for the branch + def currentCommit = sh(script: "git rev-parse ${branch}", returnStdout: true).trim() + + return currentCommit + } catch (Exception e) { + echo "Failed to get the latest commit for branch '${branch}' in path '${path}': ${e.message}" + return null + } + } + } +} diff --git a/src/com/meesho/utilities/nodePoolSelection.groovy b/src/com/meesho/utilities/nodePoolSelection.groovy new file mode 100644 index 0000000..ac8d4db --- /dev/null +++ b/src/com/meesho/utilities/nodePoolSelection.groovy @@ -0,0 +1,141 @@ +package com.meesho.utilities + +def run(String memory_request, String cpu_request, String priority_v2) { + echo 'Code to select node pool based on environment' + switch (env.cicd_environment) { + case 'prd': + echo 'Code to select node pool based on memory and cpu request in prd' + def mem_req_part = memory_request + def cpu_req_part = cpu_request + def priority = priority_v2 + echo "mem_req_part is ${mem_req_part}" + echo "cpu_req_part is ${cpu_req_part}" + if ( mem_req_part.contains('M') ) { + mem_req = mem_req_part.replaceAll('Mi', '') + mem_req = mem_req.replaceAll('M', '') + try { + mem_req = mem_req.toDouble() + } + catch (NumberFormatException e) { + mem_req = mem_req.toDouble() + //mem_req = mem_req.toInteger() + } + } + else if ( mem_req_part.contains('G') ) { + mem_req = mem_req_part.replaceAll('Gi', '') + mem_req = mem_req.replaceAll('G', '') + try { + mem_req = mem_req.toDouble() * 1024 + } + catch (NumberFormatException e) { + mem_req = mem_req.toDouble() * 1024 + //mem_req = mem_req.toInteger() + } + } + echo "memory_request is ${memory_request} ${mem_req}" + + if ( cpu_req_part.contains('m') ) { + cpu_req = cpu_req_part.replaceAll('m', '') + try { + cpu_req = cpu_req.toDouble() + } + catch (NumberFormatException e) { + cpu_req = cpu_req.toDouble() + //cpu_req = cpu_req.toInteger() + } + } + else { + try { + cpu_req = cpu_req_part.toDouble() * 1000 + } + catch (NumberFormatException e) { + cpu_req = cpu_req_part.toDouble() * 1000 + //cpu_req = cpu_req.toInteger() + } + } + echo "cpu_request is ${cpu_request} ${cpu_req}" + + def ratio = (mem_req / cpu_req).toDouble() + if ( cpu_req > mem_req ) { + ratio = 2 + } + //ratio = ratio.toInteger() + echo "Ratio - ${ratio}" + if(priority.equalsIgnoreCase('cp1')||priority.equalsIgnoreCase('cp2')||priority.equalsIgnoreCase('cp3')||priority.equalsIgnoreCase('up1')||priority.equalsIgnoreCase('up2')||priority.equalsIgnoreCase('up3')||priority.equalsIgnoreCase('sp1')||priority.equalsIgnoreCase('sp2')||priority.equalsIgnoreCase('sp3')){ + low_priority="lite" + if ( ratio >= 2.5 ) { + ratiovalue = "tetra" + } + else { + ratiovalue = "duo" + } + if ( cpu_req >=2200 ) { + nodename = "sumo" + } + else{ + nodename= "mega" + } + nodeSelectorvalue = "${nodename}${ratiovalue}${low_priority}" + println nodeSelectorvalue + break + } + else{ + if ( ratio > 5.5 ) { + ratiovalue = 'octa' + } + else if ( ratio >= 2.5 ) { + ratiovalue = 'tetra' + } + else { + ratiovalue = 'duo' + } + + if ( cpu_req >= 2200) { + nodevalue = 'sumo' + } + else if ( cpu_req < 2200 && cpu_req >= 1000) { + nodevalue = 'mega' + } + else { + nodevalue = 'compact' + } + + switch ( env.BU?.toLowerCase() ) { + case 'supply': + if (nodevalue == 'sumo' && (ratiovalue == 'hexa' || ratiovalue == 'octa')){ + ratiovalue = 'tetra' + } + else if (nodevalue == 'mega' && (ratiovalue == 'quad' || ratiovalue == 'octa')){ + ratiovalue = 'tetra' + } + else if (nodevalue == 'compact' && ratiovalue == 'trio'){ + ratiovalue = 'tetra' + } + break + case 'demand': + if (nodevalue == 'mega' && ratiovalue == 'quad'){ + ratiovalue = 'tetra' + } + else if (nodevalue == 'compact' && (ratiovalue == 'octa' || ratiovalue == 'trio')){ + ratiovalue = 'tetra' + } + break + } + + nodeSelectorvalue = "${nodevalue}${ratiovalue}" + break + + } + case 'int': + echo 'Shared node pool for int/pre-prod' + nodeSelectorvalue = "preprod-cost-optimized" + break + case ['dev', 'ftr', 'stg']: + echo 'Shared node pool for dev and ftr' + nodeSelectorvalue = "${env.BU}-shared" + break + default: + log.error('Unable to fetch environment') + } + return nodeSelectorvalue +} diff --git a/test/stubs/com/meesho/stages/buildDocker.groovy b/test/stubs/com/meesho/stages/buildDocker.groovy new file mode 100644 index 0000000..c1ef69a --- /dev/null +++ b/test/stubs/com/meesho/stages/buildDocker.groovy @@ -0,0 +1,2 @@ +package com.meesho.stages +class buildDocker { def run(Map config) {} } diff --git a/test/stubs/com/meesho/stages/buildGo.groovy b/test/stubs/com/meesho/stages/buildGo.groovy new file mode 100644 index 0000000..5b6e51f --- /dev/null +++ b/test/stubs/com/meesho/stages/buildGo.groovy @@ -0,0 +1,2 @@ +package com.meesho.stages +class buildGo { def run(Map config) {} } diff --git a/test/stubs/com/meesho/stages/buildGradle.groovy b/test/stubs/com/meesho/stages/buildGradle.groovy new file mode 100644 index 0000000..5eb8ab3 --- /dev/null +++ b/test/stubs/com/meesho/stages/buildGradle.groovy @@ -0,0 +1,2 @@ +package com.meesho.stages +class buildGradle { def run(Map config) {} } diff --git a/test/stubs/com/meesho/stages/buildMaven.groovy b/test/stubs/com/meesho/stages/buildMaven.groovy new file mode 100644 index 0000000..c1dfb8d --- /dev/null +++ b/test/stubs/com/meesho/stages/buildMaven.groovy @@ -0,0 +1,2 @@ +package com.meesho.stages +class buildMaven { def run(Map config) {} } diff --git a/test/stubs/com/meesho/stages/buildNode.groovy b/test/stubs/com/meesho/stages/buildNode.groovy new file mode 100644 index 0000000..5a836ea --- /dev/null +++ b/test/stubs/com/meesho/stages/buildNode.groovy @@ -0,0 +1,2 @@ +package com.meesho.stages +class buildNode { def run(Map config) {} } diff --git a/test/stubs/com/meesho/stages/buildPhp.groovy b/test/stubs/com/meesho/stages/buildPhp.groovy new file mode 100644 index 0000000..43f1945 --- /dev/null +++ b/test/stubs/com/meesho/stages/buildPhp.groovy @@ -0,0 +1,2 @@ +package com.meesho.stages +class buildPhp { def run(Map config) {} } diff --git a/test/stubs/com/meesho/stages/buildPython.groovy b/test/stubs/com/meesho/stages/buildPython.groovy new file mode 100644 index 0000000..9836d78 --- /dev/null +++ b/test/stubs/com/meesho/stages/buildPython.groovy @@ -0,0 +1,2 @@ +package com.meesho.stages +class buildPython { def run(Map config) {} } diff --git a/test/stubs/com/meesho/stages/buildRust.groovy b/test/stubs/com/meesho/stages/buildRust.groovy new file mode 100644 index 0000000..71e8701 --- /dev/null +++ b/test/stubs/com/meesho/stages/buildRust.groovy @@ -0,0 +1,2 @@ +package com.meesho.stages +class buildRust { def run(Map config) {} } diff --git a/test/stubs/com/meesho/utilities/addSSHKey.groovy b/test/stubs/com/meesho/utilities/addSSHKey.groovy new file mode 100644 index 0000000..5232a28 --- /dev/null +++ b/test/stubs/com/meesho/utilities/addSSHKey.groovy @@ -0,0 +1,5 @@ +package com.meesho.utilities + +class addSSHKey { + def create() { } +} diff --git a/test/stubs/com/meesho/utilities/buTeamMapping.groovy b/test/stubs/com/meesho/utilities/buTeamMapping.groovy new file mode 100644 index 0000000..55187c1 --- /dev/null +++ b/test/stubs/com/meesho/utilities/buTeamMapping.groovy @@ -0,0 +1,5 @@ +package com.meesho.utilities + +class buTeamMapping { + def get_team_initials(String team) { 'test' } +} diff --git a/test/stubs/com/meesho/utilities/constructTemplate.groovy b/test/stubs/com/meesho/utilities/constructTemplate.groovy new file mode 100644 index 0000000..fcbbc79 --- /dev/null +++ b/test/stubs/com/meesho/utilities/constructTemplate.groovy @@ -0,0 +1,5 @@ +package com.meesho.utilities + +class constructTemplate { + def renderTemplate(Map binding, String templateFile, String fileName) { } +} diff --git a/test/stubs/com/meesho/utilities/dockerUtilities.groovy b/test/stubs/com/meesho/utilities/dockerUtilities.groovy new file mode 100644 index 0000000..2786b6c --- /dev/null +++ b/test/stubs/com/meesho/utilities/dockerUtilities.groovy @@ -0,0 +1,5 @@ +package com.meesho.utilities + +class dockerUtilities { + def retryDockerPush(String command) {} +} diff --git a/test/stubs/com/meesho/utilities/getDockerParams.groovy b/test/stubs/com/meesho/utilities/getDockerParams.groovy new file mode 100644 index 0000000..72ac9bd --- /dev/null +++ b/test/stubs/com/meesho/utilities/getDockerParams.groovy @@ -0,0 +1,5 @@ +package com.meesho.utilities + +class getDockerParams { + def getTag(String repoName) { 'test-tag' } +} diff --git a/test/stubs/com/meesho/utilities/getYamlParameter.groovy b/test/stubs/com/meesho/utilities/getYamlParameter.groovy new file mode 100644 index 0000000..e735598 --- /dev/null +++ b/test/stubs/com/meesho/utilities/getYamlParameter.groovy @@ -0,0 +1,5 @@ +package com.meesho.utilities + +class getYamlParameter { + def getParam(String path, String fileName) { [:] } +} diff --git a/test/unit/com/meesho/stages/BuildNodeTest.groovy b/test/unit/com/meesho/stages/BuildNodeTest.groovy new file mode 100644 index 0000000..3afdbfb --- /dev/null +++ b/test/unit/com/meesho/stages/BuildNodeTest.groovy @@ -0,0 +1,32 @@ +package com.meesho.stages + +import com.lesfurets.jenkins.unit.BasePipelineTest +import org.junit.Before +import org.junit.Test + +import static org.junit.Assert.assertFalse +import static org.junit.Assert.assertTrue + +class BuildNodeTest extends BasePipelineTest { + + def script + + @Override + @Before + void setUp() { + super.setUp() + script = loadScript('src/com/meesho/stages/buildNode.groovy') + } + + @Test + void shouldSkipPm2Metrics_returnsTrue_whenExplicitlyEnabled() { + assertTrue(script.shouldSkipPm2Metrics([skip_pm2_metrics: true])) + } + + @Test + void shouldSkipPm2Metrics_returnsFalse_whenAbsentOrDisabled() { + assertFalse(script.shouldSkipPm2Metrics([:])) + assertFalse(script.shouldSkipPm2Metrics([skip_pm2_metrics: false])) + assertFalse(script.shouldSkipPm2Metrics([skip_pm2_metrics: 'false'])) + } +} diff --git a/test/unit/com/meesho/stages/BuildObjHelperTest.groovy b/test/unit/com/meesho/stages/BuildObjHelperTest.groovy new file mode 100644 index 0000000..e063183 --- /dev/null +++ b/test/unit/com/meesho/stages/BuildObjHelperTest.groovy @@ -0,0 +1,165 @@ +package com.meesho.stages + +import com.lesfurets.jenkins.unit.BasePipelineTest +import org.junit.Before +import org.junit.Test + +import static org.junit.Assert.* + +/** + * Unit tests for buildObjHelper.groovy — the build tool dispatch router. + * + * Stage classes (buildMaven, buildGo, etc.) are replaced with minimal stubs + * placed in test/unit/stubs/ so the script can be compiled without pulling + * in the full Jenkins API dependency chain. The stubs directory is prepended + * to scriptRoots so it takes precedence over the real src/ implementations. + */ +class BuildObjHelperTest extends BasePipelineTest { + + def script + + @Override + @Before + void setUp() { + super.setUp() + helper.registerAllowedMethod('echo', [String]) { /* no-op */ } + + // The catch block in run() uses env, log, and currentBuild — register them + binding.setProperty('env', [msg: '', FAILURE: 'FAILURE']) + binding.setProperty('log', [error: { msg -> /* no-op */ }]) + binding.setProperty('currentBuild', [result: null]) + + // defaultBuild() is called for unrecognised build_tool values + helper.registerAllowedMethod('defaultBuild', []) { null } + + script = loadScript('src/com/meesho/stages/buildObjHelper.groovy') + } + + // ──────────────────────────────── + // Exact-string routes + // ──────────────────────────────── + + @Test + void run_maven_routesToBuildMaven() { + assertEquals('buildMaven', script.run('maven').class.simpleName) + } + + @Test + void run_docker_routesToBuildDocker() { + assertEquals('buildDocker', script.run('docker').class.simpleName) + } + + @Test + void run_gradle_routesToBuildGradle() { + assertEquals('buildGradle', script.run('gradle').class.simpleName) + } + + @Test + void run_php_routesToBuildPhp() { + assertEquals('buildPhp', script.run('php').class.simpleName) + } + + // ──────────────────────────────── + // Pattern routes (~/^prefix-.*/) + // ──────────────────────────────── + + @Test + void run_maven21_routesToBuildMaven() { + assertEquals('buildMaven', script.run('maven-21').class.simpleName) + } + + @Test + void run_maven17_routesToBuildMaven() { + assertEquals('buildMaven', script.run('maven-17').class.simpleName) + } + + @Test + void run_python311_routesToBuildPython() { + assertEquals('buildPython', script.run('python-3.11').class.simpleName) + } + + @Test + void run_python39_routesToBuildPython() { + assertEquals('buildPython', script.run('python-3.9').class.simpleName) + } + + @Test + void run_node20_routesToBuildNode() { + assertEquals('buildNode', script.run('node-20').class.simpleName) + } + + @Test + void run_node18_routesToBuildNode() { + assertEquals('buildNode', script.run('node-18').class.simpleName) + } + + @Test + void run_go122_routesToBuildGo() { + assertEquals('buildGo', script.run('go-1.22').class.simpleName) + } + + @Test + void run_go121_routesToBuildGo() { + assertEquals('buildGo', script.run('go-1.21').class.simpleName) + } + + @Test + void run_go_bare_routesToBuildGo() { + assertEquals('buildGo', script.run('go').class.simpleName) + } + + @Test + void run_rust180_routesToBuildRust() { + assertEquals('buildRust', script.run('rust-1.80').class.simpleName) + } + + @Test + void run_rust_bare_routesToBuildRust() { + assertEquals('buildRust', script.run('rust').class.simpleName) + } + + // ──────────────────────────────── + // Default / unknown inputs + // ──────────────────────────────── + + @Test + void run_unknownTool_doesNotThrow() { + try { + script.run('unknown-tool') + } catch (Exception e) { + fail("Unknown build_tool must not throw — got: ${e.message}") + } + } + + @Test + void run_emptyString_doesNotThrow() { + try { + script.run('') + } catch (Exception e) { + fail("Empty build_tool must not throw — got: ${e.message}") + } + } + + // ──────────────────────────────── + // Pattern boundary guards + // ──────────────────────────────── + + @Test + void run_mavenWithoutHyphen_fallsToDefault() { + // 'mavenx' matches neither exact 'maven' nor ~/^maven-.*/ — falls through + try { + script.run('mavenx') + } catch (Exception e) { + fail("'mavenx' should reach defaultBuild, not throw: ${e.message}") + } + } + + @Test + void run_nodeWithoutHyphen_fallsToDefault() { + try { + script.run('nodex') + } catch (Exception e) { + fail("'nodex' should reach defaultBuild, not throw: ${e.message}") + } + } +} diff --git a/test/unit/com/meesho/stages/BuildRustTest.groovy b/test/unit/com/meesho/stages/BuildRustTest.groovy new file mode 100644 index 0000000..446d2ad --- /dev/null +++ b/test/unit/com/meesho/stages/BuildRustTest.groovy @@ -0,0 +1,70 @@ +package com.meesho.stages + +import com.lesfurets.jenkins.unit.BasePipelineTest +import groovy.text.SimpleTemplateEngine +import org.junit.Before +import org.junit.Test + +import static org.junit.Assert.assertEquals +import static org.junit.Assert.fail + +class BuildRustTest extends BasePipelineTest { + + def script + + @Override + @Before + void setUp() { + super.setUp() + script = loadScript('src/com/meesho/stages/buildRust.groovy') + } + + @Test + void getSystemPackages_returnsEmptyList_whenPackageKeyIsAbsent() { + assertEquals([], script.getSystemPackages([:], 'build_packages')) + } + + @Test + void getSystemPackages_returnsDistinctValidPackageNames_whenConfigured() { + assertEquals( + ['libpq-dev', 'libmariadb-dev'], + script.getSystemPackages([build_packages: ['libpq-dev', 'libmariadb-dev', 'libpq-dev']], 'build_packages') + ) + } + + @Test + void getSystemPackages_rejectsNonListPackageConfiguration() { + assertInvalid([build_packages: 'libpq-dev'], 'must be a YAML list') + } + + @Test + void getSystemPackages_rejectsBlankAndUnsafePackageNames() { + assertInvalid([build_packages: ['libpq-dev', '']], 'contains an invalid system package name') + assertInvalid([build_packages: ['libpq-dev; rm -rf /']], 'contains an invalid system package name') + } + + @Test + void rustDockerfile_rendersConfiguredBuildAndRuntimePackages() { + def template = new File('resources/com/meesho/rust-Dockerfile').text + def rendered = new SimpleTemplateEngine().createTemplate(template).make([ + buildRegistry: 'registry.example', + version: '1.85', + binary_name: 'example-service', + build_packages: ['libpq-dev'], + runtime_packages: ['libpq5'], + add_files: [] + ]).toString() + + assert rendered.contains('libpq-dev') + assert rendered.contains('libpq5') + } + + private void assertInvalid(Map config, String expectedMessage) { + try { + script.getSystemPackages(config, 'build_packages') + fail('Expected invalid package configuration to throw IllegalArgumentException') + } catch (IllegalArgumentException exception) { + assert exception.message.contains(expectedMessage) + } + } +} diff --git a/test/unit/com/meesho/utilities/BuTeamMappingTest.groovy b/test/unit/com/meesho/utilities/BuTeamMappingTest.groovy new file mode 100644 index 0000000..4509968 --- /dev/null +++ b/test/unit/com/meesho/utilities/BuTeamMappingTest.groovy @@ -0,0 +1,207 @@ +package com.meesho.utilities + +import com.lesfurets.jenkins.unit.BasePipelineTest +import org.junit.Before +import org.junit.Test + +import static org.junit.Assert.* + +class BuTeamMappingTest extends BasePipelineTest { + + def script + + @Override + @Before + void setUp() { + super.setUp() + script = loadScript('src/com/meesho/utilities/buTeamMapping.groovy') + } + + // ──────────────────────────────── + // validate() + // ──────────────────────────────── + + @Test + void validate_returnsNull_whenBuIsNull() { + assertNull(script.validate(null, 'devops')) + } + + @Test + void validate_returnsNull_whenTeamIsNull() { + assertNull(script.validate('central', null)) + } + + @Test + void validate_returnsNull_whenBothArgsAreNull() { + assertNull(script.validate(null, null)) + } + + @Test + void validate_returnsTrue_forValidSupplyTeams() { + assertTrue(script.validate('supply', 'experience')) + assertTrue(script.validate('supply', 'fulfilment')) + assertTrue(script.validate('supply', 'cataloging')) + assertTrue(script.validate('supply', 'payout')) + assertTrue(script.validate('supply', 'returns')) + assertTrue(script.validate('supply', 'supplier-service')) + assertTrue(script.validate('supply', 'display-ads')) + assertTrue(script.validate('supply', 'supplier-live-commerce')) + } + + @Test + void validate_returnsTrue_forValidDemandTeams() { + assertTrue(script.validate('demand', 'search')) + assertTrue(script.validate('demand', 'web')) + assertTrue(script.validate('demand', 'communications')) + assertTrue(script.validate('demand', 'discovery-platform')) + assertTrue(script.validate('demand', 'live-commerce')) + } + + @Test + void validate_returnsTrue_forValidCentralTeams() { + assertTrue(script.validate('central', 'devops')) + assertTrue(script.validate('central', 'shared')) + assertTrue(script.validate('central', 'psec')) + assertTrue(script.validate('central', 'dbe')) + } + + @Test + void validate_returnsTrue_forValidDataenggTeams() { + assertTrue(script.validate('dataengg', 'data-platform')) + assertTrue(script.validate('dataengg', 'data-intelligence')) + assertTrue(script.validate('dataengg', 'data-platform-nrt')) + assertTrue(script.validate('dataengg', 'data-platform-experimentation')) + } + + @Test + void validate_returnsTrue_forValidDatascienceTeams() { + assertTrue(script.validate('datascience', 'ml-platform')) + assertTrue(script.validate('datascience', 'for-you')) + assertTrue(script.validate('datascience', 'recommendation')) + assertTrue(script.validate('datascience', 'advertisement')) + } + + @Test + void validate_returnsTrue_forValidMcacheTeams() { + assertTrue(script.validate('mcache', 'mcache')) + assertTrue(script.validate('mcache', 'mcache-shared')) + assertTrue(script.validate('mcache', 'supplier-service')) + } + + @Test + void validate_returnsTrue_forValidInfraTeams() { + assertTrue(script.validate('infra', 'devops')) + assertTrue(script.validate('infra', 'dbe')) + } + + @Test + void validate_returnsFalse_whenTeamDoesNotBelongToBu() { + assertFalse(script.validate('supply', 'ml-platform')) + assertFalse(script.validate('demand', 'data-platform')) + assertFalse(script.validate('infra', 'search')) + assertFalse(script.validate('central', 'fulfilment')) + assertFalse(script.validate('dataengg', 'devops')) + } + + @Test + void validate_isCaseSensitive() { + // 'Supply' vs 'supply' — map key is lowercase only + try { + def result = script.validate('Supply', 'experience') + // Unknown bu key returns null → NullPointerException from .contains() + assertTrue("Expected null or NPE for wrong-case bu", result == null || result == false) + } catch (NullPointerException expected) { + // Acceptable: unknown bu key → null entry → NPE on .contains() + } + } + + // ──────────────────────────────── + // get_bu_initials() + // ──────────────────────────────── + + @Test + void getBuInitials_returnsNull_forNullInput() { + assertNull(script.get_bu_initials(null)) + } + + @Test + void getBuInitials_returnsNull_forUnknownBu() { + assertNull(script.get_bu_initials('unknown-bu')) + } + + @Test + void getBuInitials_returnsCorrect4LetterCodes_forAllKnownBUs() { + assertEquals('supl', script.get_bu_initials('supply')) + assertEquals('dmnd', script.get_bu_initials('demand')) + assertEquals('farm', script.get_bu_initials('farmiso')) + assertEquals('admn', script.get_bu_initials('admin')) + assertEquals('cntr', script.get_bu_initials('central')) + assertEquals('deng', script.get_bu_initials('dataengg')) + assertEquals('dsci', script.get_bu_initials('datascience')) + assertEquals('mche', script.get_bu_initials('mcache')) + assertEquals('infr', script.get_bu_initials('infra')) + } + + // ──────────────────────────────── + // get_team_initials() + // ──────────────────────────────── + + @Test + void getTeamInitials_returnsNull_forNullInput() { + assertNull(script.get_team_initials(null)) + } + + @Test + void getTeamInitials_returnsNull_forUnknownTeam() { + assertNull(script.get_team_initials('no-such-team')) + } + + @Test + void getTeamInitials_returnsCorrectCodes_forRepresentativeTeams() { + assertEquals('devop', script.get_team_initials('devops')) + assertEquals('srch', script.get_team_initials('search')) + assertEquals('ml', script.get_team_initials('ml-platform')) + assertEquals('xp', script.get_team_initials('experience')) + assertEquals('fnf', script.get_team_initials('fulfilment')) + assertEquals('dp', script.get_team_initials('data-platform')) + assertEquals('ds', script.get_team_initials('data-science')) + assertEquals('comms', script.get_team_initials('communications')) + assertEquals('web', script.get_team_initials('web')) + assertEquals('mche', script.get_team_initials('mcache')) + assertEquals('pay', script.get_team_initials('payout')) + assertEquals('retrn', script.get_team_initials('returns')) + assertEquals('dplay', script.get_team_initials('display-ads')) + assertEquals('drank', script.get_team_initials('discovery-ranking')) + assertEquals('slcom', script.get_team_initials('supplier-live-commerce')) + } + + // ──────────────────────────────── + // get_initials() — dispatch + // ──────────────────────────────── + + @Test + void getInitials_dispatchesToBuInitialsMap() { + assertEquals('supl', script.get_initials('bu_initials', 'supply')) + assertEquals('cntr', script.get_initials('bu_initials', 'central')) + assertEquals('infr', script.get_initials('bu_initials', 'infra')) + } + + @Test + void getInitials_dispatchesToTeamInitialsMap() { + assertEquals('devop', script.get_initials('team_initials', 'devops')) + assertEquals('srch', script.get_initials('team_initials', 'search')) + } + + @Test + void getInitials_returnsUndefinedOption_forUnknownTargetMap() { + assertEquals('Undefined option', script.get_initials('typo_map', 'supply')) + } + + @Test + void getInitials_returnsNull_whenTargetStringIsNull() { + assertNull(script.get_initials('bu_initials', null)) + assertNull(script.get_initials('team_initials', null)) + // Unknown map with null targetString also returns null (null guard fires first) + assertNull(script.get_initials('unknown_map', null)) + } +} diff --git a/test/unit/com/meesho/utilities/ConstructParamTest.groovy b/test/unit/com/meesho/utilities/ConstructParamTest.groovy new file mode 100644 index 0000000..1d9c3eb --- /dev/null +++ b/test/unit/com/meesho/utilities/ConstructParamTest.groovy @@ -0,0 +1,269 @@ +package com.meesho.utilities + +import com.lesfurets.jenkins.unit.BasePipelineTest +import org.junit.Before +import org.junit.Test + +import static org.junit.Assert.* + +/** + * Unit tests for constructParam.groovy policy-enforcement methods. + * + * Strategy: Jenkins DSL steps (git, readYaml, dir) are registered as no-op mocks. + * readYaml is overridden per-test to return controlled whitelist data. + * This isolates each method's branch-decision logic from the live GitHub clone. + */ +class ConstructParamTest extends BasePipelineTest { + + def script + + // Controlled whitelist sets — each test overrides as needed via helper.registerAllowedMethod + Set<String> sonarWhitelist = [] as Set + Set<String> appConfigDisabledWhitelist = [] as Set + Set<String> allowedNonDevelopWhitelist = [] as Set + Set<String> validateCacWhitelist = [] as Set + Set<String> multizoneDeployables = [] as Set + + @Override + @Before + void setUp() { + super.setUp() + + // No-op Jenkins DSL steps + helper.registerAllowedMethod('dir', [String, Closure]) { path, body -> body() } + helper.registerAllowedMethod('git', [Map]) { /* no-op clone */ } + helper.registerAllowedMethod('echo', [String]) { /* no-op */ } + + // readYaml returns whitelists driven by the test's set fields + helper.registerAllowedMethod('readYaml', [Map]) { args -> + String file = args.file as String + if (file.contains('skip-sonar-whitelist')) return [repos: sonarWhitelist.toList()] + if (file.contains('app-config-disabled')) return [repos: appConfigDisabledWhitelist.toList()] + if (file.contains('allowedNonDevelopPrDeploymentToInt')) return [repos: allowedNonDevelopWhitelist.toList()] + if (file.contains('ValidateCacConfig')) return [repos: validateCacWhitelist.toList()] + if (file.contains('multizone-enabled-repos')) return [multizone_enabled_deployables: multizoneDeployables.toList()] + return [repos: []] + } + + script = loadScript('src/com/meesho/utilities/constructParam.groovy') + } + + // ──────────────────────────────── + // skipSonarCheckForbidden() + // ──────────────────────────────── + + @Test + void skipSonarCheckForbidden_returnsFalse_forNonMavenBuilds() { + binding.setProperty('env', [BRANCH_NAME: 'main']) + def config = [dockerBuildVersion: 'go-1.22', repo_name: 'my-service', skip_sonar: true] + assertFalse(script.skipSonarCheckForbidden(config, ['main': 'prd'])) + } + + @Test + void skipSonarCheckForbidden_returnsFalse_forNodeBuild() { + binding.setProperty('env', [BRANCH_NAME: 'main']) + def config = [dockerBuildVersion: 'node-20', repo_name: 'my-service', skip_sonar: true] + assertFalse(script.skipSonarCheckForbidden(config, ['main': 'prd'])) + } + + @Test + void skipSonarCheckForbidden_returnsFalse_forHotfixBranch() { + binding.setProperty('env', [BRANCH_NAME: 'hotfix/fix-login']) + def config = [dockerBuildVersion: 'maven-21', repo_name: 'my-service', skip_sonar: true] + assertFalse(script.skipSonarCheckForbidden(config, ['main': 'prd'])) + } + + @Test + void skipSonarCheckForbidden_returnsFalse_forStgEnvironment() { + binding.setProperty('env', [BRANCH_NAME: 'develop']) + def config = [dockerBuildVersion: 'maven-21', repo_name: 'my-service', skip_sonar: true] + assertFalse(script.skipSonarCheckForbidden(config, ['develop': 'stg'])) + } + + @Test + void skipSonarCheckForbidden_returnsFalse_forIntEnvironment() { + binding.setProperty('env', [BRANCH_NAME: 'feature/xyz']) + def config = [dockerBuildVersion: 'maven-21', repo_name: 'my-service', skip_sonar: true] + // branch not in map → defaults to "int" + assertFalse(script.skipSonarCheckForbidden(config, ['main': 'prd'])) + } + + @Test + void skipSonarCheckForbidden_returnsFalse_whenRepoIsInSonarWhitelist() { + binding.setProperty('env', [BRANCH_NAME: 'main']) + sonarWhitelist = ['whitelisted-service'] as Set + def config = [dockerBuildVersion: 'maven-21', repo_name: 'whitelisted-service', skip_sonar: true] + assertFalse(script.skipSonarCheckForbidden(config, ['main': 'prd'])) + } + + @Test + void skipSonarCheckForbidden_returnsTrue_whenSkipSonarTrue_onMavenPrdBuild() { + binding.setProperty('env', [BRANCH_NAME: 'main']) + sonarWhitelist = [] as Set + def config = [dockerBuildVersion: 'maven-21', repo_name: 'my-service', skip_sonar: true] + assertTrue(script.skipSonarCheckForbidden(config, ['main': 'prd'])) + } + + @Test + void skipSonarCheckForbidden_returnsFalse_whenSkipSonarFalse_onMavenPrdBuild() { + binding.setProperty('env', [BRANCH_NAME: 'main']) + sonarWhitelist = [] as Set + def config = [dockerBuildVersion: 'maven-21', repo_name: 'my-service', skip_sonar: false] + assertFalse(script.skipSonarCheckForbidden(config, ['main': 'prd'])) + } + + @Test + void skipSonarCheckForbidden_returnsTrue_forMaven17PrdBuildWithSkipSonar() { + binding.setProperty('env', [BRANCH_NAME: 'main']) + sonarWhitelist = [] as Set + def config = [dockerBuildVersion: 'maven-17', repo_name: 'my-service', skip_sonar: true] + assertTrue(script.skipSonarCheckForbidden(config, ['main': 'prd'])) + } + + // ──────────────────────────────── + // appConfigDisabledForbidden() + // ──────────────────────────────── + + @Test + void appConfigDisabledForbidden_returnsFalse_whenEnvironmentIsNotStg() { + binding.setProperty('env', [CHANGE_TARGET: 'main']) + assertFalse(script.appConfigDisabledForbidden(false, 'my-service', 'prd', 'maven-21')) + } + + @Test + void appConfigDisabledForbidden_returnsFalse_whenEnvironmentIsInt() { + binding.setProperty('env', [CHANGE_TARGET: 'main']) + assertFalse(script.appConfigDisabledForbidden(false, 'my-service', 'int', 'maven-21')) + } + + @Test + void appConfigDisabledForbidden_returnsFalse_whenBuildIsGo() { + binding.setProperty('env', [CHANGE_TARGET: 'develop']) + assertFalse(script.appConfigDisabledForbidden(false, 'my-service', 'stg', 'go-1.22')) + } + + @Test + void appConfigDisabledForbidden_returnsFalse_whenBuildIsNode() { + binding.setProperty('env', [CHANGE_TARGET: 'develop']) + assertFalse(script.appConfigDisabledForbidden(false, 'my-service', 'stg', 'node-20')) + } + + @Test + void appConfigDisabledForbidden_returnsFalse_whenRepoIsInWhitelist() { + binding.setProperty('env', [CHANGE_TARGET: 'develop']) + appConfigDisabledWhitelist = ['legacy-service'] as Set + assertFalse(script.appConfigDisabledForbidden(false, 'legacy-service', 'stg', 'maven-21')) + } + + @Test + void appConfigDisabledForbidden_returnsTrue_forMavenStgBuildWithAppConfigDisabled() { + binding.setProperty('env', [CHANGE_TARGET: 'develop']) + appConfigDisabledWhitelist = [] as Set + assertTrue(script.appConfigDisabledForbidden(false, 'my-service', 'stg', 'maven-21')) + } + + @Test + void appConfigDisabledForbidden_returnsFalse_forMavenStgBuildWithAppConfigEnabled() { + binding.setProperty('env', [CHANGE_TARGET: 'develop']) + appConfigDisabledWhitelist = [] as Set + assertFalse(script.appConfigDisabledForbidden(true, 'my-service', 'stg', 'maven-21')) + } + + @Test + void appConfigDisabledForbidden_returnsTrue_forGradleStgBuildWithAppConfigDisabled() { + binding.setProperty('env', [CHANGE_TARGET: 'develop']) + appConfigDisabledWhitelist = [] as Set + assertTrue(script.appConfigDisabledForbidden(false, 'my-service', 'stg', 'gradle')) + } + + // ──────────────────────────────── + // allowedNonDevelopPrDeploymentToIntRepos() + // ──────────────────────────────── + + @Test + void allowedNonDevelopPr_returnsFalse_whenRepoNotInWhitelist() { + allowedNonDevelopWhitelist = [] as Set + assertFalse(script.allowedNonDevelopPrDeploymentToIntRepos('my-service')) + } + + @Test + void allowedNonDevelopPr_returnsTrue_whenRepoIsInWhitelist() { + allowedNonDevelopWhitelist = ['allowed-service'] as Set + assertTrue(script.allowedNonDevelopPrDeploymentToIntRepos('allowed-service')) + } + + @Test + void allowedNonDevelopPr_isCaseSensitive() { + allowedNonDevelopWhitelist = ['My-Service'] as Set + assertFalse(script.allowedNonDevelopPrDeploymentToIntRepos('my-service')) + } + + @Test + void allowedNonDevelopPr_returnsFalse_forEmptyRepoName() { + allowedNonDevelopWhitelist = [] as Set + assertFalse(script.allowedNonDevelopPrDeploymentToIntRepos('')) + } + + // ──────────────────────────────── + // ValidateCacConfigForRepo() + // ──────────────────────────────── + + @Test + void validateCacConfig_returnsTrue_whenRepoIsInWhitelist() { + binding.setProperty('env', [CHANGE_TARGET: 'develop']) + validateCacWhitelist = ['cac-enabled-service'] as Set + assertTrue(script.ValidateCacConfigForRepo(false, 'cac-enabled-service')) + } + + @Test + void validateCacConfig_returnsPassedInValue_whenRepoNotInWhitelist() { + binding.setProperty('env', [CHANGE_TARGET: 'develop']) + validateCacWhitelist = [] as Set + assertTrue(script.ValidateCacConfigForRepo(true, 'my-service')) + assertFalse(script.ValidateCacConfigForRepo(false, 'my-service')) + } + + @Test + void validateCacConfig_whitelistOverridesFalseValidateConfig() { + binding.setProperty('env', [CHANGE_TARGET: 'develop']) + validateCacWhitelist = ['forced-service'] as Set + assertTrue(script.ValidateCacConfigForRepo(false, 'forced-service')) + } + + @Test + void validateCacConfig_whitelistDoesNotOverrideTrueValidateConfig() { + // If repo is NOT in whitelist but ValidateConfig=true, it should still return true + binding.setProperty('env', [CHANGE_TARGET: 'develop']) + validateCacWhitelist = [] as Set + assertTrue(script.ValidateCacConfigForRepo(true, 'my-service')) + } + + // ──────────────────────────────── + // isMultizoneEnabled() + // ──────────────────────────────── + + @Test + void isMultizoneEnabled_returnsFalse_whenDeployableNotInList() { + multizoneDeployables = [] as Set + assertFalse(script.isMultizoneEnabled('my-deployable')) + } + + @Test + void isMultizoneEnabled_returnsTrue_whenDeployableIsInList() { + multizoneDeployables = ['critical-app'] as Set + assertTrue(script.isMultizoneEnabled('critical-app')) + } + + @Test + void isMultizoneEnabled_isExactMatch_prefixNotSufficient() { + multizoneDeployables = ['critical-app'] as Set + assertFalse(script.isMultizoneEnabled('critical')) + assertFalse(script.isMultizoneEnabled('critical-app-v2')) + } + + @Test + void isMultizoneEnabled_isCaseSensitive() { + multizoneDeployables = ['Critical-App'] as Set + assertFalse(script.isMultizoneEnabled('critical-app')) + } +} diff --git a/vars/buildDockerGroovyGke.groovy b/vars/buildDockerGroovyGke.groovy new file mode 100644 index 0000000..314eb67 --- /dev/null +++ b/vars/buildDockerGroovyGke.groovy @@ -0,0 +1,164 @@ +package com.meesho.stages + +def run(Map config){ + stage('Build docker image'){ + container('dockerpush') { + def env = "dev" //write if condition according to branch if branch is dev/staging or any other branch then master it will be dev if master prod + def ecrImagePod = config.ecrImagePod + def repo_name = config.repo_name + def dockerBuildVersion = config.dockerBuildVersion ?: "" + def version = getVersion() + def artifactId = getArtifactId() + def modules = getModules() + def excludedModules = config.excludedModules ?: [] + j = 0 + for (module in modules){ + if (excludedModules.contains(module)){ + echo "===== NOT Building container for Module - ${module} =====" + }else { + def dockerfileRef = getDockerfile("${dockerBuildVersion}","${repo_name}","${version}","${module}","NO") + j+=1 + sh """ + echo "===== ${j}. Creating Container for Module - ${module} ======" + if [ -f ${module}/target/*.jar ] + then + echo "docker build -t ${ecrImagePod}/${repo_name}/${module} ${dockerfileRef}; rm -f Dockerfile-${module}" + else + echo "No jar file found." + exit 1 + fi + """ + } + } + } + } +} + +def getCommitid(){ + return sh(returnStdout: true, script: 'git log -1 --format=%h').trim() +} + +def registry(String buildenv){ + switch(env) { + case "prod": + return 'asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/prod/' + break; + case "dev": + return 'asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622/dev/' + break; + default: + return 'asia-southeast1-docker.pkg.dev/meesho-devops-admin-0622' + } +} + +def getVersion(){ +// if (fileExists("pom.xml")) { +// return sh(returnStdout: true, script: 'xq -r .project.version pom.xml').trim() +// }else if (fileExists("package.json")) { +// return sh(returnStdout: true, script: 'jq -r .version package.json').trim() +// } + return "2.0" +} + +def getModules(){ + // if (fileExists("pom.xml")) { + // modules = sh(returnStdout: true, script: 'xq -r .project.modules.module[] pom.xml 2>/dev/null || xq -r .project.modules.module pom.xml 2>/dev/null || echo empty').trim() + // if (modules == 'empty' || modules == 'null') { + // return null + // } + // modules = modules.split("\n") + // return modules + // } + // else { + // return null + // } + return ["server"] +} + + +def getArtifactId(){ + // if (fileExists("pom.xml")) { + // return sh(returnStdout: true, script: 'xq -r .project.artifactId pom.xml').trim() + // } else if (fileExists("package.json")) { + // return sh(returnStdout: true, script: 'jq -r .name package.json').trim() + // } + return "external-payment-gateway" +} + + def getTag(){ + def buildenv = "${env.BUILD_ENV}" + echo "========get version========" + def version = getVersion() + echo "========commitID========" + def commitID = getCommitid() + echo "========tag========" + def tag = "v${version}-${commitID}" + return ${tag} +} + + + +def release(Map config){ + stage('Push docker image'){ + def repo_name = config.repo_name + def branch_name = "${env.BRANCH_NAME}" + def ecrImagePod = config.ecrImagePod + def bu = config.BU + def buildenv = "${env.BUILD_ENV}" + echo "========registry========" + def registry = registry("${buildenv}") + echo "========get version========" + def version = getVersion() + echo "========commitID========" + def commitID = getCommitid() + echo "========tag========" + echo "registry is ${registry}" + def tag = "v${version}-${commitID}" + echo "${registry}" + echo "tag is ${tag}" + def artifactId = getArtifactId() + def modules = getModules() + def excludedModules = config.excludedModules ?: [] + + log.info("########################### Pushing Docker Images ###########################") + if (modules == null){ + echo "docker tag ${ecrImagePod}/${repo_name} ${registry}/${bu}/${ecrImagePod}/${repo_name}:${tag}" + echo "gcloud auth configure-docker asia-southeast1-docker.pkg.dev --quiet" + echo "docker push ${registry}/${bu}/${ecrImagePod}/${repo_name}:${tag}" + }else { + j = 0 + for (module in modules){ + if (excludedModules.contains(module)){ + echo "===== NOT Pushing container for Module - ${common} =====" + }else { + j+=1 + sh """ + echo "===== ${j}. Pushing Container for Module - ${module} ======" + echo "docker tag ${ecrImagePod}/${repo_name}/${module} ${registry}/${bu}/${ecrImagePod}/${repo_name}/${module}:${tag}" + echo "gcloud auth configure-docker asia-southeast1-docker.pkg.dev --quiet" + echo "docker push ${registry}/${bu}/${ecrImagePod}/${repo_name}/${module}:${tag}" + """ + } + } + } + + } +} + +def getDockerfile(String dockerBuildVersion, String repo_name, String version, String module, String doBuild) { + if (dockerBuildVersion) { + if(dockerBuildVersion == "maven-3.3-jdk-8"){ + def scriptcontents = libraryResource "org/meesho/templates/${dockerBuildVersion}.sh" + writeFile file: "${dockerBuildVersion}.sh", text: scriptcontents + sh "chmod a+x ./${dockerBuildVersion}.sh;./${dockerBuildVersion}.sh ${module} ${version} ${doBuild} ${repo_name}" + return "-f Dockerfile-${module} ." + }else if(dockerBuildVersion == "node-12.22"){ + def scriptcontents = libraryResource "org/meesho/templates/${dockerBuildVersion}.sh" + writeFile file: "${dockerBuildVersion}.sh", text: scriptcontents + sh "chmod a+x ./${dockerBuildVersion}.sh;./${dockerBuildVersion}.sh ${module}" + return "-f Dockerfile-${module} ." + } + } else { + return "-f Dockerfile ." + } +} diff --git a/vars/buildPipeline.groovy b/vars/buildPipeline.groovy new file mode 100644 index 0000000..27cf3bc --- /dev/null +++ b/vars/buildPipeline.groovy @@ -0,0 +1,43 @@ +import com.meesho.stages.checkOut +import com.meesho.stages.buildObjHelper +import com.meesho.stages.notify +import com.meesho.stages.securityScan +import com.meesho.stages.automationTest +import com.meesho.utilities.constructParam + +def call(Map param) { + def constructParam = new constructParam() + def checkObj = new checkOut() + def buildObjHelper = new buildObjHelper() + def buildObj = buildObjHelper.run(param.build_tool) + def notify = new notify() + def securityScan = new securityScan() + def automationTest = new automationTest() + node('slave02') { + env.msg = 'Job Passed' + timestamps { + ansiColor('xterm') { + try { + constructParam.run(param) + log.info(param) + checkObj.run(param) + // securityScan.run(param) + buildObj.run(param) + if (param.containsKey('run_automation') && env.CHANGE_ID) { + automationTest.run(param) + } + } + catch (Exception e) { + if (env.msg == 'Job Passed') { + log.error(e.toString()) + currentBuild.result = env.FAILURE + env.msg = 'Job Failed. Error: ' + e.toString() + } + } + finally { + notify.run(param) + } + } + } + } +} diff --git a/vars/cdHookRunner.groovy b/vars/cdHookRunner.groovy new file mode 100644 index 0000000..5c2e037 --- /dev/null +++ b/vars/cdHookRunner.groovy @@ -0,0 +1,195 @@ +// cdHookRunner — standalone Jenkins job that runs turbo-turtle deploy-phase +// CI/CD hooks (pre_deploy / post_deploy) in an isolated agent pod. +// +// turbo-turtle triggers this parameterized job during the CD workflow (see +// TriggerCdHookPipelineActivity). It checks out the target repo, reads the +// deploy-phase hooks declared under `environment.<TT_ENV>.hooks.<HOOK_PHASE>` in +// the per-app deployment file (deployments/<APP_NAME>.yaml), executes them via +// the shared runHooks execution core, and POSTs an aggregate completion callback +// to /api/v1/cd/hook/callback which signals the waiting workflow. A blocking hook +// failure makes success=false so the workflow can fail the deploy; advisory +// (blocking:false) failures keep success=true. +// +// Required parameters (Jenkins job params, read from params/env): +// REPO_URL, TT_REPO, APP_NAME, BRANCH, COMMIT, TT_ENV, HOOK_PHASE, +// IMAGE_TAG, TT_IS_HOTFIX, TT_PR_NUMBER, TT_WORKFLOW_ID, TT_RUN_ID +def call(Map jobParams = [:]) { + String repoUrl = param('REPO_URL', jobParams) + String repoName = param('TT_REPO', jobParams) + String appName = param('APP_NAME', jobParams) + String branch = param('BRANCH', jobParams) + String commit = param('COMMIT', jobParams) + String ttEnv = param('TT_ENV', jobParams) + String phase = param('HOOK_PHASE', jobParams) + String workflowID = param('TT_WORKFLOW_ID', jobParams) + String runID = param('TT_RUN_ID', jobParams) + + if (!repoUrl && repoName) { + repoUrl = "https://github.com/Meesho/${repoName}" + } + // TT_WORKFLOW_ID / TT_RUN_ID are required: the completion callback correlates + // to the waiting Temporal run by them. If either is missing (e.g. incomplete + // job-param registration), fail before checkout/hook execution rather than run + // hooks whose callback can never be matched (workflow would wait to timeout). + if (!repoName || !appName || !ttEnv || !phase || !workflowID || !runID) { + error("cdHookRunner: TT_REPO, APP_NAME, TT_ENV, HOOK_PHASE, TT_WORKFLOW_ID and TT_RUN_ID are required parameters.") + } + // Make env available to stageName / the hook context. + env.cicd_environment = ttEnv + + String podyaml = "org/meesho/${env.INFRA_ENV ?: 'prd'}-pod.yaml" + podTemplate(yaml: libraryResource(podyaml)) { + node(POD_LABEL) { + container('devops-tools') { + boolean success = true + String errMsg = '' + try { + stage(stageName("cd-hook checkout: ${repoName}")) { + deleteDir() + dir(repoName) { + checkout([ + $class: 'GitSCM', + branches: [[name: commit ?: "*/${branch}"]], + userRemoteConfigs: [[url: repoUrl, credentialsId: 'svc-devops-meesho']], + extensions: [[$class: 'CloneOption', shallow: false, noTags: false]], + ]) + } + } + + def hooks = resolveHooks(repoName, appName, ttEnv, phase) + if (!hooks) { + log.info("cdHookRunner: no ${phase} hooks declared for env ${ttEnv} in ${repoName}/deployments/${appName}.yaml — nothing to run.") + } else { + List<String> ctxEnv = deployPhaseContextEnv(repoName, ttEnv, jobParams) + runHooks.executeHooks(repoName, hooks, phase, ctxEnv) + } + } catch (Exception e) { + success = false + errMsg = e.toString() + log.error("cdHookRunner: ${phase} hook run failed for ${repoName} (env ${ttEnv}): ${errMsg}") + } finally { + postCdHookCallback(repoName, ttEnv, phase, success, errMsg, workflowID, runID) + } + if (!success) { + // Surface as a build failure too (the workflow reads the callback, + // but a red build aids debugging). + currentBuild.result = 'FAILURE' + error("cdHookRunner: ${phase} hooks failed for ${repoName}: ${errMsg}") + } + } + } + } +} + +// resolveHooks reads the per-app deployment file (deployments/<appName>.yaml) and +// returns the deploy-phase hook list for env+phase (or null). Deploy-phase hooks +// live in deployment.yaml, not config.yaml. +def resolveHooks(String repoName, String appName, String ttEnv, String phase) { + dir(repoName) { + String deployFile = "deployments/${appName}.yaml" + if (!fileExists(deployFile)) { + error("cdHookRunner: ${deployFile} not found in ${repoName}.") + } + def deployment = readYaml file: deployFile + def envBlock = deployment?.environment?.get(ttEnv) + def hooksBlock = envBlock?.hooks + if (!(hooksBlock instanceof Map)) { + return null + } + def hooks = hooksBlock[phase] + return (hooks instanceof List && !hooks.isEmpty()) ? hooks : null + } +} + +// deployPhaseContextEnv builds the TT_* context contract from the job params. +def deployPhaseContextEnv(String repoName, String ttEnv, Map jobParams) { + return [ + "TT_REPO_NAME=${repoName}", + "TT_ENV=${ttEnv}", + "TT_EVENT=push", + "TT_IS_HOTFIX=${param('TT_IS_HOTFIX', jobParams) ?: 'false'}", + "TT_BRANCH=${param('BRANCH', jobParams) ?: ''}", + "TT_TARGET_BRANCH=", + "TT_PR_NUMBER=${param('TT_PR_NUMBER', jobParams) ?: ''}", + "TT_COMMIT_SHA=${param('COMMIT', jobParams) ?: ''}", + "TT_IMAGE_TAG=${param('IMAGE_TAG', jobParams) ?: ''}", + ] +} + +// postCdHookCallback POSTs the aggregate result to turbo-turtle, which signals +// the waiting CD workflow. Env-routed exactly like the Jenkins CI callback. +def postCdHookCallback(String repoName, String ttEnv, String phase, boolean success, String errMsg, String workflowID, String runID) { + String baseUrl + switch (ttEnv) { + case 'prd': + case 'int': + baseUrl = 'http://turbo-turtle.meeshogcp.in' + break + default: + baseUrl = 'http://turbo-turtle.admin.meeshogcp.in' + } + Map payload = [ + repo_name : repoName, + env : ttEnv, + phase : phase, + success : success, + error : errMsg, + build_url : env.BUILD_URL ?: '', + workflow_id: workflowID, + run_id : runID, + ] + String url = baseUrl + "/api/v1/cd/hook/callback" + String jsonFilePath = "cd_hook_callback_${env.BUILD_NUMBER}.json" + int maxAttempts = 3 + // The callback is turbo-turtle's ONLY completion signal for the waiting run. + // Retry delivery (bounded, with connect/request timeouts) and, if every attempt + // fails, fail the job so the failure is visible — otherwise the job goes green + // while the workflow waits until its backstop timeout. Duplicate delivery is + // safe: turbo-turtle correlates callbacks by workflow/run/repo/env/phase. + try { + writeFile(file: jsonFilePath, text: writeJSON(returnText: true, json: payload)) + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + boolean delivered = false + try { + final def(String response, String code) = sh( + returnStdout: true, + script: """ + curl -s --connect-timeout 10 --max-time 30 -X POST \\ + -H 'Content-Type: application/json' \\ + -w '\\n%{response_code}' \\ + $url \\ + -d @$jsonFilePath + """ + ).trim().tokenize("\n") + if (code == "200") { + log.info("cdHookRunner: callback delivered (attempt ${attempt}/${maxAttempts})") + delivered = true + } else { + log.warn("cdHookRunner: callback attempt ${attempt}/${maxAttempts} failed code=${code} response=${response}") + } + } catch (Exception e) { + log.warn("cdHookRunner: callback attempt ${attempt}/${maxAttempts} error: ${e}") + } + if (delivered) { + return + } + if (attempt < maxAttempts) { + sleep(time: attempt * 5, unit: 'SECONDS') + } + } + error("cdHookRunner: callback POST to ${url} failed after ${maxAttempts} attempts; turbo-turtle will not receive completion for repo=${repoName} env=${ttEnv} phase=${phase}") + } finally { + sh(script: "rm -f ${jsonFilePath}", returnStatus: true) + } +} + +// param reads a job parameter, preferring an explicit map, then params, then env. +def param(String key, Map jobParams) { + if (jobParams?.containsKey(key)) { + return jobParams[key]?.toString() + } + if (params?.containsKey(key) && params[key] != null) { + return params[key].toString() + } + return env[key]?.toString() +} diff --git a/vars/cloudFunctionCICD.groovy b/vars/cloudFunctionCICD.groovy new file mode 100644 index 0000000..d830fa9 --- /dev/null +++ b/vars/cloudFunctionCICD.groovy @@ -0,0 +1,30 @@ + +pipeline { + agent none + + environment { + K8S_LABEL = 'cloud-function-cicd-agent' + GITHUB_CRED = 'cicd-github-app' + } + + podTemplate(yaml: libraryResource('org/meesho/pod-cloud-function.yaml')) { + node(POD_LABEL) { + container('devops-tools') { + cloudFunctionCICDFlow() + } + } + } +} + +def cloudFunctionCICDFlow() { + stages { + stage { + steps { + script { + sh 'ls -al' + echo 'Hello World' + } + } + } + } +} diff --git a/vars/createEKSconfigs.groovy b/vars/createEKSconfigs.groovy new file mode 100644 index 0000000..d0c92a8 --- /dev/null +++ b/vars/createEKSconfigs.groovy @@ -0,0 +1,24 @@ +import com.meesho.stages.helmGenerator +import com.meesho.stages.validateBuTeam + +def call(Map params) { + podTemplate(yaml: libraryResource("org/meesho/${env.INFRA_ENV}-pod.yaml")) { + node(POD_LABEL) { + container('devops-tools') { + timestamps { + ansiColor('xterm') { + env.GITHUB_CRED = 'svc-devops-meesho' + // Initialize objects + def validateObj = new validateBuTeam() + def hemlGenObj = new helmGenerator() + + // validate parameters + validateObj.run(params.bu, params.team, params.module) + // Create helm file + hemlGenObj.run(params) + } + } + } + } + } +} diff --git a/vars/eksCICD.groovy b/vars/eksCICD.groovy new file mode 100644 index 0000000..13d9272 --- /dev/null +++ b/vars/eksCICD.groovy @@ -0,0 +1,150 @@ +import com.meesho.stages.hotFix +import com.meesho.stages.checkOut +import com.meesho.stages.buildObjHelper +import com.meesho.stages.notify +import com.meesho.utilities.getYamlParameter +import com.meesho.utilities.constructParam +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatterBuilder + +def call(Map repo) { + ansiColor('xterm') { + def allowedUsers = ['turbo-turtle', 'ringmaster-bot', 'yeleswaram.teja@meesho.com', 'mahak.jain@meesho.com', 'naveen.vellingiri@meesho.com', 'vignesh.ganesan@meesho.com', 'siddharth.g@meesho.com','aryaman.parida@meesho.com','shatwik.pandey@meesho.com' ,'toolchain-jenkins'] + def userId = null + + try { + def cause = currentBuild.rawBuild.getCause(hudson.model.Cause.UserIdCause) + if (cause) { + userId = cause.getUserId() + } + } catch (Exception e) { + log.error("Failed to get user ID: ${e.message}") + } + + if (userId && !allowedUsers.contains(userId)) { + echo "\u001B[1;31m========================================\n[ERROR] Build triggered by unauthorized user: ${userId}\n\nPlease use Ringmaster to trigger builds and deployments: https://ringmaster.meeshogcp.in/applications/cicd/home\n========================================\u001B[0m" + error("Build triggered by unauthorized user: ${userId}.") + } else if (!userId) { + echo "\u001B[1;31m========================================\n[ERROR] Could not determine the user who triggered the build\n\nThis might be a scheduled or system-triggered build.\n========================================\u001B[0m" + error("Could not determine the user who triggered the build") + } else { + echo "Starting the build....." + } + } + + env.STARTTIME = getDateTime() + env.FAILURE = 'FAILURE' + echo "CLOUD_PROVIDER : ${env.CLOUD_PROVIDER}" + echo "INFRA_ENV : ${env.INFRA_ENV}" + switch (env.CLOUD_PROVIDER) { + case 'GCP': + echo 'Running on GCP' + gcpInfra(repo) + break + case 'AWS': + echo 'Running on AWS' + awsInfra(repo) + break + default: + log.error('Not running on AWS or GCP') + currentBuild.result = env.FAILURE + def msg = 'Job Failed. Error: Not running on AWS or GCP' + notify.run(msg) + } +} + +def gcpInfra (Map repo) { + def isSidecarNeeded = repo.get('useSidecar', false) + def yamlName = isSidecarNeeded ? "${env.INFRA_ENV}-sidecar-pod.yaml" : "${env.INFRA_ENV}-pod.yaml" + def podyaml = "org/meesho/${yamlName}" + echo "Architecture Check: useSidecar=${isSidecarNeeded}. Loading ${podyaml}" + podTemplate(yaml: libraryResource(podyaml)) { + node(POD_LABEL) { + container('devops-tools') { + commonCICDFlow(repo) + } + } + } +} + +def awsInfra (Map repo) { + node('EKS') { + commonCICDFlow(repo) + } +} + +def commonCICDFlow (Map repo) { + def constructParam = new constructParam() + def ymlObj = new getYamlParameter() + def checkObj = new checkOut() + def buildObjHelper = new buildObjHelper() + def hotFixObj = new hotFix() + def notify = new notify() + def param = [:] + def msg = 'Job Passed' + env.msg = msg + env.error_msg_to_db = '' + + timestamps { + ansiColor('xterm') { + try { + env.deploymentStartTime = new Date().format('yyyy-MM-dd HH:mm:ss') + checkObj.run(repo) + param = ymlObj.getParam(repo.repo_name) + def maintainer = param.maintainer ?: 'jenkins-user' + def buildObj = buildObjHelper.run(param.build_tool) + constructParam.run(param) + param['skip_notify'] = env.skip_notify + log.info(param) + hotFixObj.run(param.repo_name) + // Pre-build hooks: user scripts declared under + // environment.<env>.hooks.pre_build in the repo's config.yaml. + // constructParam has merged the resolved env block into param, + // so param.hooks holds the current environment's hooks. A + // blocking hook failure fails the build before the docker build. + runHooks(param, 'pre_build') + buildObj.run(param) + // Post-build hooks: run after a successful build. + runHooks(param, 'post_build') + // Auto-trigger the standalone ai-blitz-jobs (coverage-only) job + // once the per-repo CI has succeeded. Internally gated on + // PR / hot-fix / toolchain / mainline-branch (see helper), and + // fire-and-forget — coverage observability must never block + // or fail this build. Repos opt out via skip_coverage_trigger + // in config.yaml. + triggerCoverageOnly(param) + } + catch (org.jenkinsci.plugins.workflow.steps.FlowInterruptedException e) { + currentBuild.result = 'ABORTED' + log.info(currentBuild.result) + env.msg = 'Job Aborted' + env.error_msg_to_db = 'Job ABORTED by the User' + } + catch (Exception e) { + if (env.msg == msg) { + log.error(e.toString()) + currentBuild.result = env.FAILURE + env.msg = 'Job Failed. Error: ' + e.toString() + } + } + finally { + env.deploymentEndTime = new Date().format('yyyy-MM-dd HH:mm:ss') + notify.run(param) + } + } + } +} + +@NonCPS +def getDateTime() { + // Get the current date and time in IST + def currentDateTime = ZonedDateTime.now() + // Create a formatter for the desired pattern + def formatter = new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd'T'HH:mm:ss") + .appendOffset('+HH:mm', '+00:00') + .toFormatter() + // Format the current date and time using the formatter + def formattedDateTime = currentDateTime.format(formatter) + return formattedDateTime +} diff --git a/vars/gcpMigration.groovy b/vars/gcpMigration.groovy new file mode 100644 index 0000000..fdc54a5 --- /dev/null +++ b/vars/gcpMigration.groovy @@ -0,0 +1,278 @@ +import com.meesho.utilities.gitActions +import com.meesho.utilities.buTeamMapping +import com.meesho.utilities.getYamlParameter +import com.meesho.utilities.constructTemplate +import com.meesho.stages.multiBranchPipeline +import com.meesho.stages.deployArgoCD + +def call(Map params){ + podTemplate(yaml: libraryResource('org/meesho/pod.yaml')) { + node(POD_LABEL) { + container('devops-tools') { + timestamps{ + ansiColor("xterm"){ + env.GITHUB_CRED = 'svc-devops-meesho' + def gitObj = new gitActions() + def bu_team_obj = new buTeamMapping() + def constructObj = new constructTemplate() + def application_info_map = [:] + def repo_bu = "" + stage('App Repo: cloning the repo'){ + gitObj.clone("${WORKSPACE}", params.repo_name, params.repo_main_branch) + } + + stage('APP Repo: Create Jenkinsfile(gcp-main)'){ + if (!params.skip_jenkinsfile){ + def repo_branch_name = "gcp-main" + def config = ["repo_name":params.repo_name] + gitObj.branchCheckOut(params.repo_name, repo_branch_name) + dir(params.repo_name){ + sh "chmod 777 Jenkinsfile" + constructObj.renderTemplate(config, 'Jenkinsfile', 'Jenkinsfile') + gitObj.add('.', 'Jenkinsfile') + } + def commit_status = gitObj.codeCommit(repo_name, repo_branch_name, 'Generating build and deployments files') + if (commit_status == 0) { + gitObj.codePush(repo_name, repo_branch_name) + } + } + else { + log.info("skipping this stage") + } + } + + stage('Helm Repo: Update values_property file'){ + def yaml_obj = new getYamlParameter() + gitObj.clone("${WORKSPACE}","devops-helm-charts","main") + def helm_repo_name = "devops-helm-charts" + def helm_checkout_branch = "gcp-migration-${params.repo_name}-${env.BUILD_NUMBER}" + def repo_branch_name = "gcp-main" + gitObj.branchCheckOut("devops-helm-charts", helm_checkout_branch) + gitObj.branchCheckOut(params.repo_name, repo_branch_name) + def build_config = yaml_obj.getParam(params.repo_name) + + if (build_config['copy_file'] || (build_config['environment'] && build_config['environment']['prd']['copy_file'])){ + // voilating naming convention willing to avoid confilct with deployment variables + log.info("Checking copy_file path") + def buSrt = bu_team_obj.get_bu_initials(build_config["bu"]) + def teamSrt = bu_team_obj.get_team_initials(build_config["team"]) + def copy_path = (build_config['copy_file'])? build_config.copy_file.path : build_config.environment.prd.copy_file.path + if (copy_path.startsWith("gs:")){ + log.info("Copy file path already pointing to gcs") + } else { + def file_name = copy_path.split('/')[-1] + def new_path = "gs://gcs-${buSrt}-${teamSrt}-config-prd/${params.repo_name}/${file_name}" + if(build_config.copy_file){ + build_config.copy_file.path = new_path + } else { + build_config.environment.prd.copy_file.path = new_path + } + dir(params.repo_name){ + sh "chmod 777 config.yaml" + writeYaml file: 'config.yaml', data: build_config, overwrite: true + } + gitObj.add(params.repo_name, 'config.yaml') + def config_commit = gitObj.codeCommit(repo_name, repo_branch_name, 'Changed copy_file path to gcs') + if (config_commit == 0) { + gitObj.codePush(repo_name, repo_branch_name) + } + log.warning("Path for copy_file has been changed to gcs. Please upload the contents to ${new_path} from ${copy_path}") + } + } + if (build_config.containsKey('environment')) { + Map envrionment_config = build_config['environment'].collectEntries { key, value -> "prd".matches(key)? value: [:]} + build_config.remove('environment') + build_config.putAll(envrionment_config) + } + repo_bu = build_config.bu + env.skip_user_input = params.skip_userinput + def wait_obj = new deployArgoCD() + def deployment_order = build_config.deployment_order + timeout(unit: 'SECONDS', time: 300) { + userInput = wait_obj.wait_for_user_input(build_config.deployment_order) + } + if (userInput == '') { + log.info('No Deployments selected. Running remaining steps.') + return + } + else if (!userInput.contains('All')) { + deployment_order = userInput.split(',') as List + } + + for (deployment in deployment_order){ + def application_config = yaml_obj.getParam(params.repo_name,"deployments/${deployment}.yaml") + def bu_short = bu_team_obj.get_bu_initials(application_config["bu"]) + def BU = application_config["bu"] + def team_short = bu_team_obj.get_team_initials(application_config["team"]) + //def app_name_short = application_config.app_name.substring(0, Math.min(originalString.length(), 15)) + def app_name_short = application_config.app_name + def values_version = env.cicd_environment == 'prd' ? 'values_v3' : 'values_v2' + def value_properties_path = "${values_version}/${bu_short}/${team_short}/${deployment}/values_properties.yaml" + def value_properties_content = yaml_obj.getParam(helm_repo_name,value_properties_path) + def canary_default = ['progressDeadlineSeconds': 300,'analysisInterval': '120s','analysisThreshold': 5,'analysisMaxWeight': 5,'analysisStepWeight': 5,'analysisMetrics':['thresholdRangeMin': 0.99,'interval': '1m'],'skipAnalysis': true] + + // Set/change value_properties values for migration + value_properties_content['as_min']= 1 + log.info("Setting canary skipAnalysis to true") + if (value_properties_content['canary']){ + value_properties_content['canary']['skipAnalysis'] = true + } else { + value_properties_content['canary'] = canary_default + } + log.info("Canary skipAnalysis has been set to true") + log.info("set min and nodeselector") + if (value_properties_content['ingress_class']!= "contour-external"){ + if(value_properties_content['host']){ + value_properties_content.host = application_config['app_name']+".prd.meesho.int" + } + else if(value_properties_content.hosts){ + for (host_arr in value_properties_content.hosts){ + host_arr.host = application_config['app_name']+".prd.meesho.int" + } + } + if (application_config.priority_v2 == "up0" || application_config.priority_v2 == "sp0" || application_config.priority_v2 == "cp0"){ + value_properties_content.ingress_class = "contour-internal-0" + } + else{ + value_properties_content.ingress_class = "contour-internal-1" + } + } + else if (value_properties_content['ingress_class'] == "external"){ + value_properties_content.ingress_class = "contour-external" + } + log.info("set host") + if (value_properties_content.serviceAccount){ + value_properties_content.serviceAccount.annotations.remove('eks.amazonaws.com/role-arn') + value_properties_content.serviceAccount.annotations['iam.gke.io/gcp-service-account'] = "sa-${bu_short}-prd-${app_name_short}@meesho-${BU}-prd-0622.iam.gserviceaccount.com" + } + log.info("set service account") + dir(helm_repo_name){ + sh "chmod 777 ${value_properties_path}" + writeYaml file: value_properties_path, data: value_properties_content, overwrite: true + } + + def host = (value_properties_content.host)? value_properties_content.host : value_properties_content.hosts[0].host + log.info(host) + application_info_map[application_config.app_name]=[ + "bu": application_config.bu, + "team": application_config.team, + "host": host, + "ingress_class": value_properties_content.ingress_class + ] + log.info(application_info_map) + gitObj.add(helm_repo_name,value_properties_path) + } + commit_status = gitObj.codeCommit(helm_repo_name, helm_checkout_branch, "Modified all application of ${params.repo_name} repo") + if (commit_status == 0) { + try{ + gitObj.codePush(helm_repo_name, helm_checkout_branch) + } + catch(Exception e){ + log.error("Error in code push") + return + } + try{ + pr_num = gitObj.createPR(params.repo_name, helm_repo_name, "main", helm_checkout_branch, "Merge ${params.repo_name} changes") + gitObj.mergePR(helm_repo_name, pr_num, helm_checkout_branch) + gitObj.deleteBranch(helm_repo_name, "main", helm_checkout_branch) + } + catch(Exception ex){ + log.error("error in code push") + log.error(ex) + log.info("Deleting the branch") + gitObj.deleteBranch(helm_repo_name, "main", helm_checkout_branch) + } + } + } + stage('Jenkins: Create Jenkins job'){ + def job_obj = new multiBranchPipeline() + job_obj.applicationOnboard(repo_bu,params.repo_name) + log.info("Jenkins job created") + } + // stage('GCP: Create DNS record'){ + // def zone="pvt-meesho-admin-prd-meesho-int" + // def project="meesho-admin-prd-0622" + // def app_name_list = application_info_map.collect{ it.key } + // log.info(app_name_list) + // for (app_name in app_name_list) { + // def host=application_info_map[app_name].host + // def ingress_class=application_info_map[app_name].ingress_class + // def BU=application_info_map[app_name].bu + // def dns_status= sh(script: "gcloud dns --project=${project} record-sets describe ${host} --zone=${zone} --type='CNAME' ",returnStatus: true) + // if( dns_status != 0 ){ + // dns_status = sh(script: "gcloud dns --project=${project} record-sets create ${host} --zone=${zone} --type=CNAME --ttl='300' --rrdatas='${ingress_class}.${BU}.prd.prd.meesho.int.' ",returnStatus: true) + // if(dns_status == 0 ){ + // log.info("DNS recordSet ${host} created successfully") + // } + // else{ + // log.info("DNS recordset failed with status ${dns_status}") + // } + // } + // } + + // } + // stage('GCP: Create workload identity'){ + // def bu_short=bu_team_obj.get_bu_initials(application_info_map[app_name].bu) + // def BU = application_info_map[app_name].bu + // def project="meesho-${BU}-prd-0622" + // def app_name_list = application_info_map.collect{ it.key } + // for (app_name in app_name_list){ + // if (params.provide_service_role){ + // //def app_name_short= app_name.substring(0, Math.min(originalString.length(), 15)) + // def app_name_short=app_name + // def sa = "sa-${bu_short}-prd-${app_name_short}" + // sh(script: "gcloud iam service-accounts create ${sa} --display-name='service account for ${app_name}' --project=${project} ") + // sh(script: "gcloud iam service-accounts add-iam-policy-binding --role roles/iam.workloadIdentityUser --member 'serviceAccount:${sa}@meesho-${BU}-prd-0622.iam.gserviceaccount.com' --project=meesho-${BU}-prd-0622") + // sh(script: "gcloud iam service-accounts add-iam-policy-binding --role roles/${params.SA_role} --member 'serviceAccount:${sa}@meesho-${BU}-prd-0622.iam.gserviceaccount.com' --project=meesho-${BU}-prd-0622") + // } + // } + // } + stage('CoreDNS: Add entry for svc-svc'){ + def yaml_obj = new getYamlParameter() + def coredns_repo_name = "devops-infra-helm-charts" + def coredns_checkout_branch = "gcp-migration-${params.repo_name}-${env.BUILD_NUMBER}" + gitObj.clone("${WORKSPACE}",coredns_repo_name,"main") + gitObj.branchCheckOut(coredns_repo_name, coredns_checkout_branch) + def coredns_path= "helm-templates/coredns/values.yaml" + def coredns_content = yaml_obj.getParam(coredns_repo_name,coredns_path) + def app_name_list = application_info_map.collect{ it.key } + for (app_name in app_name_list) { + def host=application_info_map[app_name].host + def ingress_class=application_info_map[app_name].ingress_class + def BU=application_info_map[app_name].bu + coredns_content.rewrites[host]="${ingress_class}-${BU}-prd" + log.info("coredns entry for ${host} is set") + dir(coredns_repo_name){ + sh "chmod 777 ${coredns_path}" + writeYaml file: coredns_path, data: coredns_content, overwrite: true + } + gitObj.add(coredns_repo_name,coredns_path) + } + commit_status = gitObj.codeCommit(coredns_repo_name, coredns_checkout_branch, "Modified all application of ${params.repo_name} repo") + if (commit_status == 0) { + try{ + gitObj.codePush(coredns_repo_name, coredns_checkout_branch) + } + catch(Exception e){ + log.error("Error in code push") + return + } + try{ + pr_num = gitObj.createPR(params.repo_name, coredns_repo_name, "main", coredns_checkout_branch, "Merge ${params.repo_name} changes") + gitObj.mergePR(coredns_repo_name, pr_num, coredns_checkout_branch) + gitObj.deleteBranch(coredns_repo_name, "main", coredns_checkout_branch) + } + catch(Exception ex){ + log.error("error in code push") + log.error(ex) + log.info("Deleting the branch") + gitObj.deleteBranch(coredns_repo_name, "main", coredns_checkout_branch) + } + } + } + } + } + } + } + } +} diff --git a/vars/gkeCICD.groovy b/vars/gkeCICD.groovy new file mode 100644 index 0000000..3f1f638 --- /dev/null +++ b/vars/gkeCICD.groovy @@ -0,0 +1,202 @@ +import com.meesho.stages.buildDocker +import com.meesho.stages.notify + + +def Codecall(Map stepParams) { + def buildDocker = new buildDocker() + def notify = new notify() + checkOutCode() + config = readYaml file: "${stepParams.file}" + def maintainer = config.maintainer ?: "jenkins-user" + def msg = "Job Passed" + try{ + mvnBuild() + dockerBuild(config) + } + catch(Exception e){ + log.error(e.toString()) + currentBuild.result = env.FAILURE + msg = "Job Failed. Error: "+ e.toString() + } + finally{ + log.info(msg) + notify.run(msg,maintainer) + } + +} + +def checkOutCode () { + stage('Checking Out Code') { + checkout scm + } +} + +def mvnBuild() { + stage('Maven Build') { + container('maven') { + sh "mvn clean install -DskipTests" + } + } + } + +def Podcall(Map stepParams) { + podTemplate( + yaml: libraryResource('org/meesho/pod.yaml')) { + node(POD_LABEL) { + Codecall(file: "$stepParams.file") + } + } +} + +def dockerBuild(Map config) { + // container('dockerpush') { + buildDockerGroovyGke.run(config) + // buildDockerGroovyGke.release(config) + // } +} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +// container('tools') { +// stage('Dockerfile creation') { +// if (config.dockerfilePresent == "Yes"){ +// sh ''' docker build -t "${config.gcrAddress}/${config.repo_name} -f ${config.dockerFileName}" ''' + +// } +// else{ +// def artifactId = config.artifactId +// def version = config.version +// def doBuild = "NO" +// sh "ls -al" +// echo "chmod after creating dockerfile" +// sh "cd server ; ls" +// // sh "cp /root/.m2/repository/com/meesho/payment/gateway/server/2.9.1/server-2.9.1.jar ." +// def dockerfile = buildDocker.getDockerfile(config.dockerBuildVersion, config.repo_name, version, artifactId, doBuild) +// echo "ls after creating dockerfile" +// sh "ls " + +// def tag = config.version + +// sh "gcloud auth configure-docker asia-southeast1-docker.pkg.dev --quiet" +// sh "docker build -t asia-southeast1-docker.pkg.dev/supply-poc-351106/meesho-devops/external-payment-gateway:${tag} ${dockerfile}" +// sh "docker push asia-southeast1-docker.pkg.dev/supply-poc-351106/meesho-devops/external-payment-gateway:${tag}" + + + +// // echo "docker build -t ${config.gcrImagePod}/${config.repo_name} ${dockerfile}" +// // echo "docker tag ${config.gcrImagePod}/${config.repo_name} ${config.gcrAddress}/${config.gcrImagePod}/${config.repo_name}:${tag}" +// // echo "docker push ${config.gcrAddress}/${config.gcrImagePod}/${config.repo_name}:${tag}" +// } +// } +// } +// } diff --git a/vars/log.groovy b/vars/log.groovy new file mode 100644 index 0000000..f5965a2 --- /dev/null +++ b/vars/log.groovy @@ -0,0 +1,24 @@ +def RED = '\033[1;31m' +def GREEN = '\033[1;32m' +def BLUE = '\033[1;34m' +def BLACK = '\033[0m' +def CYAN_BG = '\033[1;46m' +def YELLOW_BG = '\033[1;43m' + +def info(message) { + def GREEN = '\033[1;32m' + def BLACK = '\033[0m' + echo "${GREEN}INFO: ${message}${BLACK}" +} + +def warning(message) { + def RED = '\033[1;31m' + def BLACK = '\033[0m' + echo "${RED}WARNING: ${message}${BLACK}" +} + +def error(message){ + def RED = '\033[1;31m' + def BLACK = '\033[0m' + echo "${RED}ERROR: ${message}${BLACK}" +} diff --git a/vars/onlyPushtoJfrog.groovy b/vars/onlyPushtoJfrog.groovy new file mode 100644 index 0000000..e8df1d4 --- /dev/null +++ b/vars/onlyPushtoJfrog.groovy @@ -0,0 +1,115 @@ +pipeline { + agent { + kubernetes { + yamlFile "resources/org/meesho/${env.INFRA_ENV}-pod.yaml" + defaultContainer 'devops-tools' + } + } + + environment { + GITHUB_CRED = 'svc-devops-meesho' + } + + parameters { + string(name: 'repo_name', defaultValue: '', trim: true, description: 'Please enter repo name.') + string(name: 'branch_name', defaultValue: 'main', trim: true, description: 'Please enter branch name.') + choice(name: 'jdk_version', choices: ['jdk8', 'jdk11', 'jdk17', 'jdk21', 'jdk25'], description: 'Please select JDK Version. For adding new JDK Version, please contact DevOps Team.') + booleanParam(name: 'skip_tests', defaultValue: true, description: 'This option will skip tests while mvn build.') + booleanParam(name: 'sonar_scan', defaultValue: true, description: 'This option will perform static analysis of code via SonarQube.') + booleanParam(name: 'sub_modules', defaultValue: false, description: 'This option will enable the use of Git submodules.') + } + + stages { + stage('Validate Params') { + steps { + script { + echo '####################### Validating Parameters ###########################' + echo "Repo Name: ${params.repo_name}" + echo "Branch Name: ${params.branch_name}" + echo "JDK Version: ${params.jdk_version}" + echo "Skip Tests: ${params.skip_tests}" + echo "Sonar Scan: ${params.sonar_scan}" + echo "Sub module: ${params.sub_modules}" + } + } + } + stage('Build') { + steps { + ansiColor('xterm') { + script { + sh 'whoami' + def extra_args = '' + if (env.INFRA_ENV == 'prd') { + jfrog_repo = '-DuseProdRepo=true' + } + else { + jfrog_repo = '-DuseTestRepo=true' + } + if (skip_tests.toBoolean()) { + extra_args = '-DskipTests' + } + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh "git clone -b ${branch_name} https://github.com/Meesho/${repo_name}.git" + } + if (sub_modules.toBoolean()) { + dir("${repo_name}") { + withCredentials([gitUsernamePassword(credentialsId: "${env.GITHUB_CRED}", gitToolName: 'git-tool')]) { + sh(script:'git submodule init && git submodule update && git submodule update --recursive --remote') + } + } + } + + def effective_jdk = params.jdk_version + dir("${repo_name}") { + if (fileExists('config.yaml')) { + def config = readYaml file: 'config.yaml' + def dv = config?.dockerBuildVersion?.toString() ?: '' + def matcher = (dv =~ /maven-.*-jdk-(\d+)/) + if (matcher) { + effective_jdk = "jdk${matcher[0][1]}" + echo "Using JDK from config.yaml dockerBuildVersion: ${dv} -> ${effective_jdk}" + } + } + } + if (effective_jdk == 'jdk25') { + env.JAVA_HOME = '/usr/lib/jvm/java-25-openjdk-amd64/' + } + else if (effective_jdk == 'jdk21') { + env.JAVA_HOME = '/usr/lib/jvm/java-21-openjdk-amd64/' + } + else if (effective_jdk == 'jdk17') { + env.JAVA_HOME = '/usr/lib/jvm/java-17-openjdk-amd64/' + } + else if (effective_jdk == 'jdk11') { + env.JAVA_HOME = '/usr/lib/jvm/java-11-openjdk-amd64/' + } + else { + env.JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64/' + } + echo "Resolved JAVA_HOME: ${env.JAVA_HOME} (effective_jdk: ${effective_jdk})" + dir("${repo_name}") { + echo '####################### Building Artifacts ###########################' + sh """ + pwd + ls -al + mvn -v + mvn clean install ${extra_args} + """ + if (sonar_scan.toBoolean()) { + withSonarQubeEnv('sonarqube-test') { + echo '####################### Performing Sonar Scan ###########################' + sh(script: "JAVA_HOME='${env.JAVA_HOME}' mvn sonar:sonar -Dsonar.branch.name=${branch_name} -Dsonar.projectName=${repo_name}") + } + } + else { + echo '####################### Skipping Sonar Scan ###########################' + } + echo '####################### Pushing Artifacts to Jfrog ###########################' + sh(script: "mvn package deploy -DskipTests=${skip_tests} ${jfrog_repo}") + } + } + } + } + } + } +} diff --git a/vars/runHooks.groovy b/vars/runHooks.groovy new file mode 100644 index 0000000..8c0e16b --- /dev/null +++ b/vars/runHooks.groovy @@ -0,0 +1,160 @@ +// runHooks — generic per-environment CI/CD hook runner (execution core). +// +// Executes user-provided scripts (shell or python) that live in the service +// repo and are declared under `environment.<env>.hooks.<phase>` in the repo's +// config.yaml. The platform stays generic: it only locates, contextualises +// (via TT_* env vars) and runs each script — all use-case logic lives in the +// script itself. +// +// Build-phase entry (call from commonCICDFlow, repo already checked out): +// runHooks(param, 'pre_build') // before docker build +// runHooks(param, 'post_build') // after a successful build +// +// Deploy-phase re-use (from cdHookRunner, after it checks out the repo): +// runHooks.executeHooks(repoName, hooks, phase, ctxEnv) +// +// By the time the build-phase entry runs, constructParam has merged the +// resolved environment block into `param` top-level, so the current env's hooks +// are at `param.hooks`. The repo is checked out into `${repoName}`, so scripts +// run with CWD at the repo root and can reach any repo file. +def call(Map param, String phase) { + def hooksBlock = param?.hooks + if (!(hooksBlock instanceof Map)) { + return + } + def hooks = hooksBlock[phase] + if (!(hooks instanceof List) || hooks.isEmpty()) { + return + } + + String repoName = param?.repo_name + if (!repoName) { + error("runHooks: param.repo_name is missing — cannot locate hook scripts.") + } + + executeHooks(repoName, hooks, phase, buildPhaseContextEnv(repoName, phase)) +} + +// executeHooks runs each hook spec in `hooks` inside the repo working directory, +// injecting `ctxEnv` (the TT_* context contract) plus a per-hook TT_HOOK_NAME. +// Reused by both the build-phase entry and the deploy-phase cdHookRunner. +def executeHooks(String repoName, def hooks, String phase, List<String> ctxEnv) { + hooks.eachWithIndex { hook, idx -> + runOneHook(repoName, phase, idx, hook, ctxEnv) + } +} + +// runOneHook executes a single hook spec inside the repo working directory. +def runOneHook(String repoName, String phase, int idx, def hook, List<String> ctxEnv) { + String name = (hook?.name ?: "${phase}-${idx}").toString() + String script = hook?.script?.toString() + String interpreter = hook?.interpreter?.toString() + String requirements = hook?.requirements?.toString() + // Blocking by default; only an explicit `blocking: false` demotes to advisory. + boolean blocking = !(hook?.blocking?.toString() == 'false') + int timeoutSeconds = 600 + if (hook?.timeout_seconds) { + try { timeoutSeconds = hook.timeout_seconds.toString().toInteger() } catch (ignored) { timeoutSeconds = 600 } + } + + if (!script) { + error("runHooks: ${phase}[${idx}] '${name}' has no 'script' path in config.yaml.") + } + assertRepoRelative(script, "${phase}[${idx}] '${name}' script") + if (requirements) { + assertRepoRelative(requirements, "${phase}[${idx}] '${name}' requirements") + } + + stage(stageName("${phase}: ${name}")) { + dir(repoName) { + if (!fileExists(script)) { + error("runHooks: ${phase} hook '${name}' script not found in repo: ${script}") + } + + List<String> hookEnv = [] + hookEnv.addAll(ctxEnv) + hookEnv.add("TT_HOOK_PHASE=${phase}") + hookEnv.add("TT_HOOK_NAME=${name}") + String runCmd = buildRunCommand(script, interpreter, requirements) + + log.info("runHooks: executing ${phase} hook '${name}' (${script}), blocking=${blocking}, timeout=${timeoutSeconds}s") + withEnv(hookEnv) { + try { + timeout(time: timeoutSeconds, unit: 'SECONDS') { + sh(script: runCmd) + } + } catch (Exception e) { + if (blocking) { + log.error("runHooks: blocking ${phase} hook '${name}' failed: ${e}") + throw e + } + // Advisory hook — record and continue without failing the build. + log.warning("runHooks: advisory ${phase} hook '${name}' failed (non-blocking): ${e}") + currentBuild.description = (currentBuild.description ? currentBuild.description + " | " : "") + "hook(${name}) advisory-failed" + } + } + } + } +} + +// buildRunCommand resolves the interpreter (explicit > extension > shebang) and, +// when a python requirements file is given, provisions an ephemeral venv. +def buildRunCommand(String script, String interpreter, String requirements) { + String interp = interpreter + if (!interp) { + if (script.endsWith('.py')) { + interp = 'python3' + } else if (script.endsWith('.sh')) { + interp = 'bash' + } + } + + boolean isPython = (interp == 'python3' || interp == 'python' || script.endsWith('.py')) + if (requirements && isPython) { + String py = interp ?: 'python3' + return """ + set -e + ${py} -m venv .tt_hook_venv + . .tt_hook_venv/bin/activate + pip install --quiet --disable-pip-version-check -r ${requirements} + ${py} ${script} + """.stripIndent().trim() + } + + if (interp) { + return "set -e\n${interp} ${script}" + } + // No interpreter resolved — rely on the script's shebang. + return "set -e\nchmod +x ${script}\n./${script}" +} + +// buildPhaseContextEnv builds the standard TT_* env-var contract for build-phase +// hooks from the pipeline env already populated by constructParam / the GitHub +// Branch Source plugin. +def buildPhaseContextEnv(String repoName, String phase) { + boolean isPR = (env.CHANGE_ID ? true : false) + String branch = isPR ? (env.CHANGE_BRANCH ?: env.BRANCH_NAME ?: '') : (env.BRANCH_NAME ?: '') + boolean isHotfix = (env.hot_fix == 'true' || env.hot_fix == true) + return [ + "TT_REPO_NAME=${repoName}", + "TT_ENV=${env.cicd_environment ?: ''}", + "TT_EVENT=${isPR ? 'pr' : 'push'}", + "TT_IS_HOTFIX=${isHotfix}", + "TT_BRANCH=${branch}", + "TT_TARGET_BRANCH=${env.CHANGE_TARGET ?: ''}", + "TT_PR_NUMBER=${env.CHANGE_ID ?: ''}", + "TT_COMMIT_SHA=${env.GIT_COMMIT ?: ''}", + "TT_IMAGE_TAG=${env.image_tag ?: ''}", + ] +} + +// assertRepoRelative rejects absolute paths and parent-directory traversal so a +// hook can only execute code that lives inside the checked-out repo. +def assertRepoRelative(String p, String what) { + if (p.startsWith('/')) { + error("runHooks: ${what} path '${p}' must be repo-relative, not absolute.") + } + if (p == '..' || p.startsWith('../') || p.contains('/../')) { + error("runHooks: ${what} path '${p}' must not traverse outside the repo ('..').") + } +} diff --git a/vars/stageName.groovy b/vars/stageName.groovy new file mode 100644 index 0000000..f5db242 --- /dev/null +++ b/vars/stageName.groovy @@ -0,0 +1,16 @@ +def call(String description) { + // Initialize or increment a global step counter kept in the environment + def current = (env.STEP_COUNTER ?: '0').trim() + int next + try { + next = current.toInteger() + 1 + } catch (Throwable ignored) { + next = 1 + } + env.STEP_COUNTER = next.toString() + + // Derive environment name from common env vars + def stageEnv = env.cicd_environment ?: env.INFRA_ENV ?: env.BUILD_ENV ?: env.ENVIRONMENT ?: 'ftr' + + return "[${stageEnv}] [Step ${env.STEP_COUNTER}] ${description}" +} diff --git a/vars/triggerCoverageOnly.groovy b/vars/triggerCoverageOnly.groovy new file mode 100644 index 0000000..394f6a2 --- /dev/null +++ b/vars/triggerCoverageOnly.groovy @@ -0,0 +1,92 @@ +// Best-effort fire of the standalone ai-blitz-jobs (coverage-only) Jenkins +// job after a successful per-repo CI build. ai-blitz-jobs runs the +// coverage-only.Jenkinsfile from devops-lib@coverage-only-pipeline against +// the repo+branch we just built, so coverage and test-quality metrics flow +// to novaviz without anyone having to trigger the job manually. +// +// Gates — skip when triggering would waste compute or pollute the data: +// - PR builds (CHANGE_ID set; coverage on PRs is noise) +// - hot-fix builds (env.hot_fix == true) +// - toolchain image rebuilds (env.INFRA_ENV == 'toolchain') +// - explicit opt-out (skip_coverage_trigger: true in config.yaml) +// - non-develop branches (current policy: ONLY 'develop' fires. main / +// master / gcp-main / gcp-master are +// deploy-only branches at Meesho — tests +// already ran on the develop merge that +// produced their content, so re-running +// coverage on them would be duplicate +// spend. Expand this allowlist if a repo +// ships from a non-develop branch.) +// +// Failure is swallowed — observability must never block prod CI. The +// ai-blitz-jobs Jenkins job being renamed/disabled/missing logs a warning +// and continues; the parent build stays green. +def call(Map param) { + if (env.CHANGE_ID) { + log.info("triggerCoverageOnly: skipping PR build (CHANGE_ID=${env.CHANGE_ID}).") + return + } + if (env.hot_fix == 'true' || env.hot_fix == true) { + log.info("triggerCoverageOnly: skipping hot-fix build.") + return + } + if ((env.INFRA_ENV ?: '') == 'toolchain') { + log.info("triggerCoverageOnly: skipping toolchain (image-rebuild) build.") + return + } + if (param?.skip_coverage_trigger?.toString() == 'true') { + log.info("triggerCoverageOnly: skipping — repo opted out via config.yaml skip_coverage_trigger=true.") + return + } + + String branch = (env.BRANCH_NAME ?: '').trim() + Set<String> allowed = ['develop'] + if (!allowed.contains(branch)) { + log.info("triggerCoverageOnly: skipping branch '${branch}' (only ${allowed} triggers coverage today).") + return + } + + String repoName = param?.repo_name + if (!repoName) { + log.warning("triggerCoverageOnly: param.repo_name missing — cannot construct REPO_URL. Skipping.") + return + } + // Meesho convention: every backend service lives at github.com/Meesho/<repo>. + // ai-blitz-jobs' coverage-only.Jenkinsfile takes the full URL as REPO_URL. + String repoUrl = "https://github.com/Meesho/${repoName}" + + // Absolute path. Per-repo CI jobs may live inside Jenkins folders + // (e.g. /Meesho/order-service); a relative 'ai-blitz-jobs' would try + // the parent folder first and 404 there. The leading slash anchors at + // the Jenkins root, where ai-blitz-jobs lives (confirmed by the URL + // shape http://jenkins-dev.../job/ai-blitz-jobs/<N>/). + String targetJob = '/ai-blitz-jobs' + + try { + log.info("triggerCoverageOnly: launching ${targetJob} for ${repoName} @ ${branch} (fire-and-forget).") + // The `build` step is in-process Jenkins RPC — no API tokens or + // credentials needed. It runs with the parent build's identity + // (UpstreamCause), and ai-blitz-jobs' coverage-only.Jenkinsfile + // calls coverageOnly() directly (NOT eksCICD), so the + // allowedUsers gate in eksCICD doesn't apply to this path. Bot + // and upstream-triggered builds both work. + // wait:false → parent build queues the downstream and moves on + // immediately. It never reads the downstream result, so `propagate` + // is a no-op here and intentionally omitted. The try/catch below is + // what shields the parent from step-level errors (job not found, + // bad params, queue full). + build( + job: targetJob, + parameters: [ + string(name: 'REPO_URL', value: repoUrl), + string(name: 'BRANCH', value: branch) + ], + wait: false, + quietPeriod: 0 + ) + } catch (Exception e) { + // ai-blitz-jobs renamed/disabled/queue-blocked/whatever — log and + // continue. We never block the parent build on observability. + log.warning("triggerCoverageOnly: build(job: '${targetJob}', …) threw: ${e}. Continuing — parent CI stays green.") + } +}