From 7b835eb35ddca9d34651229fc22230f9eedabcab Mon Sep 17 00:00:00 2001 From: John Sundarraj Date: Fri, 15 Dec 2023 14:52:07 +0530 Subject: [PATCH] Revamp Jenkins pipeline script * Modified CI/CD workflow in the Jenkins pipeline script. * Integrated Jenkins pipeline with Docker and Ansible to execute CI/CD pipeline. * Removed Docker, Supervisor, and other configurations from the repository. --- .gitignore | 7 +- README.md | 2 +- conf/security-limits.conf | 1 - data/.gitignore | 4 - docker/compose/main.yml | 39 ------ docker/image/context/jenkins/.gitignore | 4 - .../image/context/jenkins/secret/.gitignore | 4 - docker/image/jenkins.dockerfile | 53 ------- downloads/readme.txt | 1 - log/.gitignore | 4 - pipeline/crm-stack.jenkinsfile | 75 ---------- pipeline/crm.jenkinsfile | 118 ++++++++++++++++ supervisor/conf/jenkins.ini | 7 - supervisor/conf/main.conf | 129 ------------------ supervisor/conf/sshd.ini | 4 - system/.gitignore | 4 - 16 files changed, 120 insertions(+), 336 deletions(-) delete mode 100644 conf/security-limits.conf delete mode 100644 data/.gitignore delete mode 100644 docker/compose/main.yml delete mode 100644 docker/image/context/jenkins/.gitignore delete mode 100644 docker/image/context/jenkins/secret/.gitignore delete mode 100644 docker/image/jenkins.dockerfile delete mode 100644 downloads/readme.txt delete mode 100644 log/.gitignore delete mode 100644 pipeline/crm-stack.jenkinsfile create mode 100644 pipeline/crm.jenkinsfile delete mode 100644 supervisor/conf/jenkins.ini delete mode 100644 supervisor/conf/main.conf delete mode 100644 supervisor/conf/sshd.ini delete mode 100644 system/.gitignore diff --git a/.gitignore b/.gitignore index fa9c468..47768d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,4 @@ -downloads/jdk-* -downloads/*.rpm -downloads/*.deb -downloads/*.tar.gz -downloads/*.zip -downloads/*.jar secret/*.pem secret/*.pub +secret/node* tmp/* diff --git a/README.md b/README.md index c97ba3e..433ef5d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ # Kickstart Jenkins -The ultimate aim of this Jenkins starter-kit is to help you run build/test of sample CRM app using Jenkins freestyle and pipeline projects. +Our Jenkins starter-kit repository contains Jenkins pipeline templates and other configurations needed to implement CI/CD workflow for workloads using Jenkins. diff --git a/conf/security-limits.conf b/conf/security-limits.conf deleted file mode 100644 index ab90892..0000000 --- a/conf/security-limits.conf +++ /dev/null @@ -1 +0,0 @@ -jenkins - nofile 8192 diff --git a/data/.gitignore b/data/.gitignore deleted file mode 100644 index c8b3bb1..0000000 --- a/data/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# ignore everything in this directory. -* -# except this file. -!.gitignore diff --git a/docker/compose/main.yml b/docker/compose/main.yml deleted file mode 100644 index d60d9dc..0000000 --- a/docker/compose/main.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: '3.8' -services: - jenkins-master: - image: sloopstash/jenkins:v1 - container_name: jenkins-master - command: "/usr/bin/supervisord -c /etc/supervisord.conf" - hostname: jenkins-master - ports: - - "8080:8080" - - "50002:50002" - volumes: - - ../../supervisor/conf/main.conf:/etc/supervisord.conf - - ../../supervisor/conf/sshd.ini:/etc/supervisord.d/sshd.ini - - ../../supervisor/conf/jenkins.ini:/etc/supervisord.d/jenkins.ini - - ../../conf/security-limits.conf:/etc/security/limits.d/10-jenkins-defaults.conf - - ../../data:/opt/jenkins/data - - ../../log:/opt/jenkins/log - - ../../system:/opt/jenkins/system - networks: - jenkins-net: - ipv4_address: 100.5.1.10 - jenkins-slave: - image: sloopstash/jenkins:v1 - container_name: jenkins-slave - command: "/usr/bin/supervisord -c /etc/supervisord.conf" - hostname: jenkins-slave - volumes: - - ../../supervisor/conf/main.conf:/etc/supervisord.conf - - ../../supervisor/conf/sshd.ini:/etc/supervisord.d/sshd.ini - - /opt/kickstart-ansible:/opt/kickstart-ansible - depends_on: - - jenkins-master - networks: - jenkins-net: - ipv4_address: 100.5.1.20 -networks: - jenkins-net: - external: - name: jenkins-net diff --git a/docker/image/context/jenkins/.gitignore b/docker/image/context/jenkins/.gitignore deleted file mode 100644 index 409b213..0000000 --- a/docker/image/context/jenkins/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory. -* -# Except this file. -!.gitignore diff --git a/docker/image/context/jenkins/secret/.gitignore b/docker/image/context/jenkins/secret/.gitignore deleted file mode 100644 index 409b213..0000000 --- a/docker/image/context/jenkins/secret/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory. -* -# Except this file. -!.gitignore diff --git a/docker/image/jenkins.dockerfile b/docker/image/jenkins.dockerfile deleted file mode 100644 index 1219afa..0000000 --- a/docker/image/jenkins.dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -# Docker image to use. -FROM sloopstash/amazonlinux:v1 - -# Install OpenSSH server and Git. -RUN yum install -y openssh-server passwd git - -# Configure OpenSSH server. -RUN set -x \ - && mkdir /var/run/sshd \ - && ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N '' \ - && sed -i 's/PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config - -# Configure OpenSSH user. -RUN set -x \ - && mkdir /root/.ssh \ - && touch /root/.ssh/authorized_keys \ - && touch /root/.ssh/config \ - && echo -e "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile=/dev/null" >> /root/.ssh/config \ - && chmod 400 /root/.ssh/config -ADD secret/node.pub /root/.ssh/authorized_keys - -# Switch work directory. -WORKDIR /tmp - -# Install Oracle JDK. -COPY jdk-8u131-linux-x64.rpm ./ -RUN set -x \ - && rpm -Uvh jdk-8u131-linux-x64.rpm \ - && rm -rf jdk-8u131-linux-x64.rpm - -# Create system user for Jenkins. -RUN set -x \ - && useradd -m -s /bin/bash -d /usr/local/lib/jenkins jenkins - -# Install Jenkins. -COPY jenkins.war ./ -RUN set -x \ - && cp jenkins.war /usr/local/lib/jenkins/ \ - && chown -R jenkins:jenkins /usr/local/lib/jenkins \ - && rm jenkins.war -RUN set -x \ - && mkdir /opt/jenkins \ - && mkdir /opt/jenkins/data \ - && mkdir /opt/jenkins/log \ - && mkdir /opt/jenkins/system \ - && touch /opt/jenkins/system/process.pid \ - && chown -R jenkins:jenkins /opt/jenkins - -# Switch work directory. -WORKDIR / - -# Cleanup history. -RUN history -c diff --git a/downloads/readme.txt b/downloads/readme.txt deleted file mode 100644 index 3d8881c..0000000 --- a/downloads/readme.txt +++ /dev/null @@ -1 +0,0 @@ -- Place your downloaded packages here. diff --git a/log/.gitignore b/log/.gitignore deleted file mode 100644 index c8b3bb1..0000000 --- a/log/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# ignore everything in this directory. -* -# except this file. -!.gitignore diff --git a/pipeline/crm-stack.jenkinsfile b/pipeline/crm-stack.jenkinsfile deleted file mode 100644 index 69c0285..0000000 --- a/pipeline/crm-stack.jenkinsfile +++ /dev/null @@ -1,75 +0,0 @@ -pipeline { - agent { - node { - label 'slave' - } - } - environment { - ANSIBLE_CONFIG = '/opt/kickstart-ansible/ansible.cfg' - } - stages { - stage('Build') { - steps { - sh 'easy_install pip' - sh 'pip install ansible==2.8.2' - sh 'pip install pytest' - git ( - url: 'https://github.com/sloopstash/kickstart-flask', - branch: 'master' - ) - sh 'pip install -r requirements.txt' - } - } - stage('Test') { - steps { - sh '/usr/bin/pytest --junitxml=reports.xml script/test/main.py' - } - post { - always { - junit 'reports.xml' - } - success { - input 'Congrats! The build/test of CRM App is successful. Do you want to start deployment on staging environment?' - } - } - } - stage('Deploy/Staging') { - steps { - ansiblePlaybook ( - credentialsId: 'ansible-node-ssh', - disableHostKeyChecking: true, - inventory: '/opt/kickstart-ansible/inventory/stg', - playbook: '/opt/kickstart-ansible/playbook/redis.yml', - // extras: '-e {"redis":{"version":"4.0.8"}}', - tags: 'setup, configure, start' - ) - ansiblePlaybook ( - credentialsId: 'ansible-node-ssh', - disableHostKeyChecking: true, - inventory: '/opt/kickstart-ansible/inventory/stg', - playbook: '/opt/kickstart-ansible/playbook/app.yml', - // extras: '-e {"app":{"environment":"stg"}}', - tags: 'setup, update, configure, start', - limit: 'crm-stg-app-1' - ) - } - post { - always { - emailext ( - body: "${currentBuild.currentResult}: Job ${env.JOB_NAME} build ${env.BUILD_NUMBER}\n More info at: ${env.BUILD_URL}", - recipientProviders: [[$class: 'DevelopersRecipientProvider'], [$class: 'RequesterRecipientProvider']], - to: 'devops@sloopstash.com', - subject: "Jenkins Build ${currentBuild.currentResult}: Job ${env.JOB_NAME}" - ) - } - success { - cleanWs( - deleteDirs: true, - disableDeferredWipeout: true - ) - input 'Congrats! The deployment on staging environment is successful. Do you want to start deployment on production environment?' - } - } - } - } -} diff --git a/pipeline/crm.jenkinsfile b/pipeline/crm.jenkinsfile new file mode 100644 index 0000000..84f250a --- /dev/null +++ b/pipeline/crm.jenkinsfile @@ -0,0 +1,118 @@ +pipeline { + agent any + parameters { + choice( + name:'test_environment',choices:['qaa','qab'],description:'Testing environment' + ) + } + environment { + TEST_ENVIRONMENT=params.test_environment.toUpperCase() + } + stages { + stage('CI: Download sources') { + steps { + sh( + script:'git clone https://github.com/sloopstash/kickstart-docker.git kickstart-docker', + returnStatus:true + ) + sh( + script:'git clone https://github.com/sloopstash/kickstart-ansible.git kickstart-ansible', + returnStatus:true + ) + } + } + stage('CI: Build OCI images') { + steps { + dir('kickstart-docker') { + sh 'sudo docker image build -t sloopstash/base:v1.1.1 -f image/base/1.1.1/amazon-linux-2.dockerfile image/base/1.1.1/context' + sh 'sudo docker image build -t sloopstash/nginx:v1.14.0 -f image/nginx/1.14.0/amazon-linux-2.dockerfile image/nginx/1.14.0/context' + sh 'sudo docker image build -t sloopstash/python:v2.7 -f image/python/2.7/amazon-linux-2.dockerfile image/python/2.7/context' + sh 'sudo docker image build -t sloopstash/redis:v4.0.9 -f image/redis/4.0.9/amazon-linux-2.dockerfile image/redis/4.0.9/context' + } + } + } + stage('CI: Bootstrap testing environment') { + steps { + dir('kickstart-docker') { + sh "sudo docker compose -f compose/crm.yml --env-file compose/${TEST_ENVIRONMENT}.env -p sloopstash-${params.test_environment}-crm up -d" + sh "sudo docker container exec sloopstash-${params.test_environment}-crm-app-1 pip install pytest" + } + } + } + stage('CI: Execute test cases') { + environment { + APP_SOURCE='/opt/app/source' + } + steps { + sh "sudo docker container exec --workdir ${APP_SOURCE} sloopstash-${params.test_environment}-crm-app-1 pytest --junitxml=reports.xml script/test/main.py" + } + post { + // always { + // junit 'reports.xml' + // } + success { + input 'App testing has been successful. Do you want to proceed deployment in staging environment?' + } + } + } + stage('CD: Bootstrap staging environment') { + steps { + dir('kickstart-ansible') { + sh 'sudo docker compose -f docker/compose/crm.yml --env-file docker/compose/STG.env -p sloopstash-stg-crm up -d --scale app=3 --scale nginx=2' + } + } + } + stage('CD: Execute deployment') { + steps { + dir('kickstart-ansible') { + ansiblePlaybook( + credentialsId:'ansible-node-ssh', + playbook:'playbook/redis.yml', + inventory:'inventory/stg', + tags:'setup,configure,stop,start' + ) + ansiblePlaybook( + credentialsId:'ansible-node-ssh', + playbook:'playbook/crm/app.yml', + inventory:'inventory/stg', + tags:'setup,update,configure,stop,start', + limit:'sloopstash-stg-crm-app-1' + ) + ansiblePlaybook( + credentialsId:'ansible-node-ssh', + playbook:'playbook/nginx.yml', + inventory:'inventory/stg', + tags:'setup,update,configure,stop,start', + limit:'sloopstash-stg-crm-nginx-1' + ) + } + } + post { + success { + input 'Deployment has been successful. Do you want to proceed deployment in production environment?' + } + } + } + } + post { + always { + dir('kickstart-docker') { + sh "sudo docker compose -f compose/crm.yml --env-file compose/${TEST_ENVIRONMENT}.env -p sloopstash-${params.test_environment}-crm down" + } + dir('kickstart-ansible') { + sh 'sudo docker compose -f docker/compose/crm.yml --env-file docker/compose/STG.env -p sloopstash-stg-crm down' + } + emailext( + subject:'Jenkins: ${env.JOB_NAME} - ${currentBuild.currentResult}', + body:'${env.BUILD_NUMBER} ${env.JOB_NAME} execution ${currentBuild.currentResult}.\n Know more at ${env.BUILD_URL}.', + recipientProviders:[[$class:'DevelopersRecipientProvider'],[$class:'RequesterRecipientProvider']] + ) + } + success { + cleanWs( + deleteDirs:true, + disableDeferredWipeout:true + ) + } + } +} diff --git a/supervisor/conf/jenkins.ini b/supervisor/conf/jenkins.ini deleted file mode 100644 index d12ff1a..0000000 --- a/supervisor/conf/jenkins.ini +++ /dev/null @@ -1,7 +0,0 @@ -[program:jenkins] -command=/bin/su jenkins /bin/bash -c "JENKINS_HOME=/opt/jenkins/data JENKINS_ENABLE_ACCESS_LOG=yes /usr/bin/java -Xms1024m -Xmx1024m -server -Djava.awt.headless=true -jar /usr/local/lib/jenkins/jenkins.war --logfile=/opt/jenkins/log/jenkins.log --httpPort=8080" -process_name=%(program_name)s -pidfile=/opt/jenkins/system/process.pid -numprocs=1 -autorestart=false -autostart=true diff --git a/supervisor/conf/main.conf b/supervisor/conf/main.conf deleted file mode 100644 index 0a30557..0000000 --- a/supervisor/conf/main.conf +++ /dev/null @@ -1,129 +0,0 @@ -; Sample supervisor config file. - -[unix_http_server] -file=/var/run/supervisor.sock ; (the path to the socket file) -;chmod=0700 ; sockef file mode (default 0700) -;chown=nobody:nogroup ; socket file uid:gid owner -;username=user ; (default is no username (open server)) -;password=123 ; (default is no password (open server)) - -;[inet_http_server] ; inet (TCP) server disabled by default -;port=127.0.0.1:9001 ; (ip_address:port specifier, *:port for all iface) -;username=user ; (default is no username (open server)) -;password=123 ; (default is no password (open server)) - -[supervisord] -logfile=/var/log/supervisord.log ; (main log file;default $CWD/supervisord.log) -logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) -logfile_backups=10 ; (num of main logfile rotation backups;default 10) -loglevel=info ; (log level;default info; others: debug,warn,trace) -pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) -nodaemon=true ; (start in foreground if true;default false) -minfds=1024 ; (min. avail startup file descriptors;default 1024) -minprocs=200 ; (min. avail process descriptors;default 200) -;umask=022 ; (process file creation umask;default 022) -;user=chrism ; (default is current user, required if root) -;identifier=supervisor ; (supervisord identifier, default is 'supervisor') -;directory=/tmp ; (default is not to cd during start) -;nocleanup=true ; (don't clean up tempfiles at start;default false) -;childlogdir=/tmp ; ('AUTO' child log dir, default $TEMP) -;environment=KEY=value ; (key value pairs to add to environment) -;strip_ansi=false ; (strip ansi escape codes in logs; def. false) - -; the below section must remain in the config file for RPC -; (supervisorctl/web interface) to work, additional interfaces may be -; added by defining them in separate rpcinterface: sections -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL for a unix socket -;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket -;username=chris ; should be same as http_username if set -;password=123 ; should be same as http_password if set -;prompt=mysupervisor ; cmd line prompt (default "supervisor") -;history_file=~/.sc_history ; use readline history if available - -; The below sample program section shows all possible program subsection values, -; create one or more 'real' program: sections to be able to control them under -; supervisor. - -;[program:theprogramname] -;command=/bin/cat ; the program (relative uses PATH, can take args) -;process_name=%(program_name)s ; process_name expr (default %(program_name)s) -;numprocs=1 ; number of processes copies to start (def 1) -;directory=/tmp ; directory to cwd to before exec (def no cwd) -;umask=022 ; umask for process (default None) -;priority=999 ; the relative start priority (default 999) -;autostart=true ; start at supervisord start (default: true) -;autorestart=true ; retstart at unexpected quit (default: true) -;startsecs=10 ; number of secs prog must stay running (def. 1) -;startretries=3 ; max # of serial start failures (default 3) -;exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) -;stopsignal=QUIT ; signal used to kill process (default TERM) -;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) -;user=chrism ; setuid to this UNIX account to run the program -;redirect_stderr=true ; redirect proc stderr to stdout (default false) -;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO -;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stdout_logfile_backups=10 ; # of stdout logfile backups (default 10) -;stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) -;stdout_events_enabled=false ; emit events on stdout writes (default false) -;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO -;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stderr_logfile_backups=10 ; # of stderr logfile backups (default 10) -;stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) -;stderr_events_enabled=false ; emit events on stderr writes (default false) -;environment=A=1,B=2 ; process environment additions (def no adds) -;serverurl=AUTO ; override serverurl computation (childutils) - -; The below sample eventlistener section shows all possible -; eventlistener subsection values, create one or more 'real' -; eventlistener: sections to be able to handle event notifications -; sent by supervisor. - -;[eventlistener:theeventlistenername] -;command=/bin/eventlistener ; the program (relative uses PATH, can take args) -;process_name=%(program_name)s ; process_name expr (default %(program_name)s) -;numprocs=1 ; number of processes copies to start (def 1) -;events=EVENT ; event notif. types to subscribe to (req'd) -;buffer_size=10 ; event buffer queue size (default 10) -;directory=/tmp ; directory to cwd to before exec (def no cwd) -;umask=022 ; umask for process (default None) -;priority=-1 ; the relative start priority (default -1) -;autostart=true ; start at supervisord start (default: true) -;autorestart=unexpected ; restart at unexpected quit (default: unexpected) -;startsecs=10 ; number of secs prog must stay running (def. 1) -;startretries=3 ; max # of serial start failures (default 3) -;exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) -;stopsignal=QUIT ; signal used to kill process (default TERM) -;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) -;user=chrism ; setuid to this UNIX account to run the program -;redirect_stderr=true ; redirect proc stderr to stdout (default false) -;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO -;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stdout_logfile_backups=10 ; # of stdout logfile backups (default 10) -;stdout_events_enabled=false ; emit events on stdout writes (default false) -;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO -;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) -;stderr_logfile_backups ; # of stderr logfile backups (default 10) -;stderr_events_enabled=false ; emit events on stderr writes (default false) -;environment=A=1,B=2 ; process environment additions -;serverurl=AUTO ; override serverurl computation (childutils) - -; The below sample group section shows all possible group values, -; create one or more 'real' group: sections to create "heterogeneous" -; process groups. - -;[group:thegroupname] -;programs=progname1,progname2 ; each refers to 'x' in [program:x] definitions -;priority=999 ; the relative start priority (default 999) - -; The [include] section can just contain the "files" setting. This -; setting can list multiple files (separated by whitespace or -; newlines). It can also contain wildcards. The filenames are -; interpreted as relative to this file. Included files *cannot* -; include files themselves. - -[include] -files = supervisord.d/*.ini \ No newline at end of file diff --git a/supervisor/conf/sshd.ini b/supervisor/conf/sshd.ini deleted file mode 100644 index 05af5c3..0000000 --- a/supervisor/conf/sshd.ini +++ /dev/null @@ -1,4 +0,0 @@ -[program:sshd] -command=bash -c "/usr/sbin/sshd -D" -process_name=%(program_name)s -autorestart=false \ No newline at end of file diff --git a/system/.gitignore b/system/.gitignore deleted file mode 100644 index c8b3bb1..0000000 --- a/system/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# ignore everything in this directory. -* -# except this file. -!.gitignore