K8s环境持续部署

1、Kubectl 发布流水线

CI流水线：

下载代码
构建单元测试
代码扫描
构建镜像
更新发布文件

CD流水线：

输入版本，从git下载部署文件；
使用kubectl发布；
使用kubectl回滚；

环境准备

jenkins

gitlab

sonarQube

harbor

使用之前的npm项目

使用之前的npm项目devops6-npm-service

如何构建，请看npm构建项目部分。

新建Dockerfile

FROM nginx:1.17.0

COPY index.html /usr/share/nginx/html/

编辑index.html文件

<h1>VERSION: main</h1>

配置好sonar参数

创建sonar-project.properties文件

# 定义唯一的关键字
sonar.projectKey=devops6-npm-service

# 定义项目名称
sonar.projectName=devops6-npm-service

# 定义项目的版本信息
sonar.projectVersion=1.0
 
# 指定扫描代码的目录位置（多个逗号分隔）
sonar.sources=src
 
# 执行项目编码
sonar.sourceEncoding=UTF-8

# 指定sonar Server
sonar.host.url=http://172.29.9.101:9000

# 认证信息
sonar.login=admin
sonar.password=Admin@123

然后提交项目代码。

创建一条新pipeline

以原devops6-maven-service_CI为基础拷贝一条新流水线devops6-npm-service_K8SCI：

⚠️ 注意：但是这里的Git选项参数一直没效果，测试不出来，很奇怪。。。因此，就直接用默认的选项参数就好。

开始进行构建

可以看待，前面几个过程都是ok的，这里上传制品过程，不需要。

创建harbor仓库

镜像命名规范：

创建仓库

修改本地 docker 服务使用 http 协议和私有仓库通信（否则会报错）

#配置可信任（如果仓库是HTTPS访问不用配置）
#在 daemon.json 中添加以下参数
[root@harbor ~]# vim /etc/docker/daemon.json #创建此文件，并写入以下内容
{
"insecure-registries": ["172.29.9.120"] 
}

#重启docker 服务
[root@harbor ~]# systemctl daemon-reload && systemctl restart docker

编写CI pipeline

创建k8sci.jenkinsfile

@Library("devops06@main") _

//import src/org/devops/xxx.groovy
def checkout = new org.devops.CheckOut()
def build = new org.devops.Build()
def sonar = new org.devops.Sonar()


//使用git 参数需要格式化
env.branchName = "${env.branchName}" - "origin/"
println(env.branchName)

pipeline {
    agent {label "build"}

    //跳过默认的代码检出功能
    options {
        skipDefaultCheckout true
    }    
    

    stages{
        stage("CheckOut"){
            steps{
                script{
                    checkout.CheckOut()
                    
                    //获取commitID
                    env.commitID = checkout.GetCommitID()
                    println("commitID: ${env.commitID}")

                    // Jenkins构建显示信息
                    currentBuild.displayName = "第${BUILD_NUMBER}次构建-${env.commitID}"
                    currentBuild.description = "构建分支名称：${env.branchName}"
                    //currentBuild.description = "Trigger by user jenkins \n branch: ${env.branchName}"
                }
            }
        }

        stage("Build"){
            steps{
                script{
                    build.Build()
                }
            }

        }        

        stage("CodeScan"){
            // 是否跳过代码扫描？
            when {
                environment name: 'skipSonar', value: 'false'
            }
            
            steps{
                script{
                    sonar.SonarScannerByPlugin()        

                }
            }
        }

         stage("ImageBuild"){
            steps{
                script{
                    //PushArtifactByPlugin()
                    //PushArtifactByPluginPOM()

                    // init package info
                    appName = "${JOB_NAME}".split('_')[0] //devops6-maven-service_CI
                    repoName = appName.split('-')[0]   //devops6
  
                    imageName = "${repoName}/${appName}"
                    imageTag = "${env.branchName}-${env.commitID}"
                    sh """
                        #登录镜像仓库
                        docker login -u admin -p Harbor12345 172.29.9.120

                        # 构建镜像
                        docker build -t 172.29.9.120/${imageName}:${imageTag} .

                        # 上传镜像
                        docker push 172.29.9.120/${imageName}:${imageTag}

                        # 删除镜像
                        sleep 2
                        docker rmi 172.29.9.120/${imageName}:${imageTag}
                    """
                }
            }
    
        }       

    }
}

在回放里运行

然后将代码提交到共享库里。

==准备k8s环境==

启动好k8s环境

在devops06机器上安装kubectl工具

cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

[root@Devops6 ~]#yum install -y kubectl-1.20.0 --disableexcludes=kubernetes

然后把k8s集群master节点下的~/.kube/config文件拷贝到devops6机器~/.kube/目录下

然后在k8s的2个节点先配置好修改本地 docker 服务使用 http 协议和私有仓库通信（否则会报错）

#配置可信任（如果仓库是HTTPS访问不用配置）
#在 daemon.json 中添加以下参数
[root@harbor ~]# vim /etc/docker/daemon.json #创建此文件，并写入以下内容
{
"insecure-registries": ["172.29.9.120"] 
}

#重启docker 服务
[root@harbor ~]# systemctl daemon-reload && systemctl restart docker

记得：只需要在node1 node2上配置就行。

自己k8s集群需不是一个ingress-controller。

这里之前已经部署好ingress-controller了。

配置下kubectl的自动补全

#安装软件包
yum install -y epel-release bash-completion

#执行命令
source /usr/share/bash-completion/bash_completion

source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc
source ~/.bashrc

==创建一个devops6-deploy-repo仓库==

创建Deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: devops6-npm-service
spec:
  replicas: 3
  revisionHistoryLimit: 3
  selector:
    matchLabels:
      app: devops6-npm-service
  template:
    metadata:
      labels:
        app: devops6-npm-service
    spec:
      containers:
      - image: 172.29.9.120/devops6/devops6-npm-service:main-ed12ce10
        name: devops6-npm-service
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: devops6-npm-service
spec:
  type: ClusterIP
  selector:
    app: devops6-npm-service
  ports:
    - name: http
      protocol: TCP
      port: 80
      targetPort: 80
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: devops6-npm-service
  annotations:
    kubernetes.io/ingress.class: nginx
spec:
  rules:
  - host: devops.test.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: devops6-npm-service
            port: 
              name: http

自己部署应用测试

创建新命名空间devops6

[root@Devops6 ~]#kubectl create ns devops6

部署应用：

[root@Devops6 ~]#kubectl  -n  devops6 apply -f Deployment.yaml 
deployment.apps/devops6-npm-service created
service/devops6-npm-service created
ingress.networking.k8s.io/devops6-npm-service created

注意：这里直接加上命名空间后，应用就会直接被部署到此命名空间了。

[root@Devops6 ~]#kubectl get po -ndevops6
NAME                                  READY   STATUS    RESTARTS   AGE
devops6-npm-service-bd4978ff9-27bpp   1/1     Running   0          32s
devops6-npm-service-bd4978ff9-clkhm   1/1     Running   0          32s
devops6-npm-service-bd4978ff9-x2sw8   1/1     Running   0          32s

配置ingress域名解析：

[root@Devops6 ~]#vim /etc/hosts
172.29.9.31 devops.test.com

[root@Devops6 ~]#kubectl get ingress -ndevops6
NAME                  CLASS    HOSTS             ADDRESS       PORTS   AGE
devops6-npm-service   <none>   devops.test.com   172.29.9.31   80      8m11s

测试效果：

[root@Devops6 ~]#curl devops.test.com
<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width,initial-scale=1.0">
    <title>vuedemo</title>
  </head>
  <body>
    <div id="app">
      <h1>VERSION: main</h1>
    </div>
    <!-- built files will be auto injected -->
  </body>
</html>

==新建devops6-npm-service版本分支，特性分支==

以main为基础新建devops6-npm-service版本分支RELEASE-1.1.1

修改index.html的内容为RELEASE-1.1.1。

运行一次CI流水线

要记得改下这里的jenkinsfile文件：

成功生成镜像：

我们再来更新一个版本的应用程序看下

[root@Devops6 ~]#vim Deployment.yaml
将
- image: 172.29.9.120/devops6/devops6-npm-service:main-ed12ce10
替换为
172.29.9.120/devops6/devops6-npm-service:RELEASE-1.1.1-7d906f68

#然后部署应用：
[root@Devops6 ~]#kubectl apply -f Deployment.yaml -ndevops6

#验证
[root@Devops6 ~]#curl devops.test.com
<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width,initial-scale=1.0">
    <title>vuedemo</title>
  </head>
  <body>
    <div id="app">
      <h1>VERSION: RELEASE-1.1.1</h1>
    </div>
    <!-- built files will be auto injected -->
  </body>
</html>
#可以看到，已经更新成功了。

我们来测试下回滚

回滚命令：

## 查看历史
kubectl rollout history deployment/anyops-devopsdocker-ui

## 查看具体某一个历史版本信息
kubectl rollout history deployment/anyops-devopsdocker-ui --revision=2

## 回滚上个版本 
kubectl rollout undo deployment/anyops-devopsdocker-ui -n anyops

## 回滚指定版本
kubectl rollout undo deployment/nginx --to-revision=2

查看当前应用版本：

[root@Devops6 ~]#kubectl rollout history deployment devops6-npm-service -ndevops6
deployment.apps/devops6-npm-service 
REVISION  CHANGE-CAUSE
1         <none>
2         <none>
#可以看到有2个历史版本

我们打算回滚到上个历史版本：

先来查看下当前应用版本：

watch -n 1 "curl devops.test.com"

watch -n 1 "curl -s devops.test.com"

开始回滚：

[root@Devops6 ~]#kubectl rollout undo deployment devops6-npm-service -ndevops6
deployment.apps/devops6-npm-service rolled back

回滚结果：

回滚成功。

这里调的是gitlab的api

调用gitlab api自动更新配置文件。

jenkins装一个插件HTTP Request

devops6-npm-service项目ProjectID为11。

创建devops6-npm-service目录

将Deployment.yaml里的image改为__IMAGE_NAME__

pipeline代码

Gitlab.groovy

package org.devops

//获取文件内容
def GetRepoFile(projectId,filePath,branchName){
   //GET /projects/:id/repository/files/:file_path/raw
   apiUrl = "/projects/${projectId}/repository/files/${filePath}/raw?ref=${branchName}"
   response = HttpReq('GET', apiUrl)
   return response
}

//更新文件内容
def UpdateRepoFile(projectId,filePath,fileContent, branchName){
    apiUrl = "projects/${projectId}/repository/files/${filePath}"
    reqBody = """{"branch": "${branchName}","encoding":"base64", "content": "${fileContent}", "commit_message": "update a new file"}"""
    response = HttpReqByPlugin('PUT',apiUrl,reqBody)
    println(response)

}

//创建文件
def CreateRepoFile(projectId,filePath,fileContent, branchName){
    apiUrl = "projects/${projectId}/repository/files/${filePath}"
    reqBody = """{"branch": "${branchName}","encoding":"base64", "content": "${fileContent}", "commit_message": "update a new file"}"""
    response = HttpReqByPlugin('POST',apiUrl,reqBody)
    println(response)

}

// 封装HTTP
def HttpReqByPlugin(reqType, reqUrl,reqBody ){
    def gitServer = "http://172.29.9.101:8076/api/v4"
    withCredentials([string(credentialsId: '5782c77d-ce9d-44e5-b9ba-1ba2097fc31d', 
                            variable: 'GITLABTOKEN')]) {
        response = httpRequest acceptType: 'APPLICATION_JSON_UTF8', 
                    consoleLogResponseBody: true, 
                    contentType: 'APPLICATION_JSON_UTF8', 
                    customHeaders: [[maskValue: false, name: 'PRIVATE-TOKEN', value: "${GITLABTOKEN}"]], 
                    httpMode: "${reqType}", 
                    url: "${gitServer}/${reqUrl}", 
                    wrapAsMultipart: false,
                    requestBody: "${reqBody}"

    }
    return response
}



//发起HTTP请求
//调用gitlab api
def HttpReq(method, apiUrl){

    withCredentials([string(credentialsId: '5782c77d-ce9d-44e5-b9ba-1ba2097fc31d', variable: 'gitlabtoken')]) {
        response = sh  returnStdout: true, 
        script: """ 
        curl --location --request ${method} \
        http://172.29.9.101:8076/api/v4/${apiUrl} \
        --header "PRIVATE-TOKEN: ${gitlabtoken}"
        """
    }

    //新增这段代码
    try {
        response = readJSON text: response - "\n" //json数据的读取方式
    } catch(e){
        println(e)
    }
    return response    
}

//获取ProjectID
def GetProjectIDByName(projectName, groupName){
    apiUrl = "projects?search=${projectName}"
    response = HttpReq("GET", apiUrl)
    if (response != []){
        for (p in response) {
            if (p["namespace"]["name"] == groupName){
                return response[0]["id"]
            }
        }
    }
}

//获取分支CommitID
def GetBranchCommitID(projectID, branchName){
    apiUrl = "projects/${projectID}/repository/branches/${branchName}"
    response = HttpReq("GET", apiUrl)
    return response.commit.short_id
}

k8sci.jenkinsfile

@Library("devops06@main") _

//import src/org/devops/xxx.groovy
def checkout = new org.devops.CheckOut()
def build = new org.devops.Build()
def sonar = new org.devops.Sonar()
def mygit = new org.devops.Gitlab()


//使用git 参数需要格式化
env.branchName = "${env.branchName}" - "origin/"
println(env.branchName)

pipeline {
    agent {label "build"}

    //跳过默认的代码检出功能
    options {
        skipDefaultCheckout true
    }    
    

    stages{
        stage("CheckOut"){
            steps{
                script{
                    checkout.CheckOut()
                    
                    //获取commitID
                    env.commitID = checkout.GetCommitID()
                    println("commitID: ${env.commitID}")

                    // Jenkins构建显示信息
                    currentBuild.displayName = "第${BUILD_NUMBER}次构建-${env.commitID}"
                    currentBuild.description = "构建分支名称：${env.branchName}"
                    //currentBuild.description = "Trigger by user jenkins \n branch: ${env.branchName}"
                }
            }
        }

        stage("Build"){
            steps{
                script{
                    build.Build()
                }
            }

        }        

        stage("CodeScan"){
            // 是否跳过代码扫描？
            when {
                environment name: 'skipSonar', value: 'false'
            }
            
            steps{
                script{
                    sonar.SonarScannerByPlugin()        

                }
            }
        }

        stage("ImageBuild"){
            steps{
                script{
                    //PushArtifactByPlugin()
                    //PushArtifactByPluginPOM()

                    // init package info
                    appName = "${JOB_NAME}".split('_')[0] //devops6-maven-service_CI
                    repoName = appName.split('-')[0]   //devops6
  
                    imageName = "${repoName}/${appName}"
                    imageTag = "${env.branchName}-${env.commitID}"
                    env.fullImageName = "172.29.9.120/${imageName}:${imageTag}"
                    sh """
                        #登录镜像仓库
                        docker login -u admin -p Harbor12345 172.29.9.120

                        # 构建镜像
                        docker build -t ${env.fullImageName} .

                        # 上传镜像
                        docker push ${env.fullImageName}

                        # 删除镜像
                        sleep 2
                        docker rmi ${env.fullImageName}
                    """
                }
            }
    
        }
        
        stage("UpdateEnvFile"){
            steps{
                script {
                    // 更新部署文件

                    projectId = 11
                    fileName = "Deployment.yaml"  //模板文件
                    branchName = "main"

                    //下载模板文件
                    fileData = mygit.GetRepoFile(projectId,fileName,branchName)
                    sh "rm -fr ${fileName}"

                    //模板文件内容保存到本地
                    writeFile file: fileName , text: fileData
                    env.deployFile = fileName
                    //替换镜像
                    sh "sed -i 's#__IMAGE_NAME__#${env.fullImageName}#g' ${env.deployFile} "
                    sh "ls -l ; cat ${fileName}"

                    //创建/更新发布文件
                    newYaml = sh returnStdout: true, script: "cat ${env.deployFile}"
                    println(newYaml)
                   
                    //更新gitlab文件内容
                    base64Content = newYaml.bytes.encodeBase64().toString()
                    
                    appName = "${JOB_NAME}".split('_')[0] //devops6-npm-service
                    env.groupName = appName.split('-')[0]   //devops6
                    env.projectName = appName

                    // 会有并行问题，同时更新报错
                    try {
                        mygit.UpdateRepoFile(projectId,"${env.projectName}%2f${env.branchName}.yaml",base64Content, "main")
                    } catch(e){
                        mygit.CreateRepoFile(projectId,"${env.projectName}%2f${env.branchName}.yaml",base64Content, "main")
                    }
                }
            }
        }              

    }
}

再次新建版本分支RELEASE-2.1.1，jenkins里记得配置下该分支名。

运行

点击approve

符合预期。

【GitOps最重要的一个特性： ==自动更新部署文件==】

更新生成版本文件

步骤：拿到 env仓库中的deployments.yaml模板文件，然后替换内容，更新到版本库。

更新对象：
- RELEASE_VERSION(镜像TAG)

       stage("UpdateEnvFile"){
            steps{
                script {
                    // 更新部署文件
      
                    projectId = 11
                    fileName = "Deployment.yaml"  //模板文件
                    branchName = "main"
      
                    //下载模板文件
                    fileData = mygit.GetRepoFile(projectId,fileName,branchName)
                    sh "rm -fr ${fileName}"
      
                    //模板文件内容保存到本地
                    writeFile file: fileName , text: fileData
                    env.deployFile = fileName
                    //替换镜像
                    sh "sed -i 's#__IMAGE_NAME__#${env.fullImageName}#g' ${env.deployFile} "
                    sh "ls -l ; cat ${fileName}"
      
                    //创建/更新发布文件
                    newYaml = sh returnStdout: true, script: "cat ${env.deployFile}"
                    println(newYaml)
                   
                    //更新gitlab文件内容
                    base64Content = newYaml.bytes.encodeBase64().toString()
                    
                    appName = "${JOB_NAME}".split('_')[0] //devops6-npm-service
                    env.groupName = appName.split('-')[0]   //devops6
                    env.projectName = appName
      
                    // 会有并行问题，同时更新报错
                    try {
                        mygit.UpdateRepoFile(projectId,"${env.projectName}%2f${env.branchName}.yaml",base64Content, "main")
                    } catch(e){
                        mygit.CreateRepoFile(projectId,"${env.projectName}%2f${env.branchName}.yaml",base64Content, "main")
                    }
                }
            }
        }              

更新后的版本文件

编写CD pipeline

创建一个CD的job devops6-npm-service_K8SCD

配置git仓库：

编写pipeline

k8scd.jenkinsfile文件

@Library("devops06@main") _ 

def mygit = new org.devops.Gitlab()

//使用git 参数需要格式化
env.branchName = "${env.branchName}" - "origin/"
println(env.branchName)

pipeline {
    agent { label "build"}
    options {
        skipDefaultCheckout true
    }
    stages{
        stage("GetManifests"){
            steps{
                script{
                   
                   //下载发布文件
                    projectId = 11
                    env.deployFile = "${env.branchName}.yaml"  //版本分支RELEASE-2.1.1.yaml
                    env.appName = "${JOB_NAME}".split('_')[0] //devops6-maven-service
                    filePath = "${env.appName}%2f${env.deployFile}" //devops6-npm-service/RELEASE-2.1.1.yaml
                    branchName = "main"
                    fileData = mygit.GetRepoFile(projectId,filePath,branchName)
                    sh "rm -fr ${env.deployFile}"
                    writeFile file: env.deployFile , text: fileData
                    sh "ls -l ; cat ${env.deployFile}"
                }
            }
        }

        stage("Deploy"){
            steps{
                script{
                    env.namespace = "${env.appName}".split('-')[0]   //devops6

                    sh """
                        ## 发布应用
                        kubectl apply -f ${env.deployFile} -n ${env.namespace}

                    """

                    // 获取应用状态
                    5.times{
                        sh "sleep 2; kubectl -n ${env.namespace} get pod | grep ${env.appName}"
                    }
                }
            }
        }

        stage("RollBack"){
            input {
                message "是否进行回滚"
                ok "提交"
                submitter ""
                parameters {
                    choice(choices: ['yes', 'no'], name: 'opts')
                }
            }
            steps{
                script{
                    switch("${opts}") {
                        case "yes":
                            sh "kubectl rollout undo deployment/${env.appName} -n ${env.namespace} "
                            break
                        case "no":
                            break
                    }
                }
            }
        }
    }
}

运行流水线

我们来提前观察下此时版本：

符合预期。

完整代码

完整代码如下：

链接：https://pan.baidu.com/s/1XFtZ0epIwgVwu0jzQHyJkA?pwd=0820 提取码：0820

2023.7.9-day9-k8s-ci-cd （kubectl和helm ci-cd）

CI/CD共享库源码前端项目（Dockerfile） K8s清单文件代码仓库 helm chart仓库

环境：

gitlab-ce:15.0.3-ce.0
jenkins:2.346.3-2-lts-jdk11
sonarqube:9.9.0-community
harbor v2.6.2

k8sci.jenkinsfile文件

@Library("devops06@main") _

//import src/org/devops/xxx.groovy
def checkout = new org.devops.CheckOut()
def build = new org.devops.Build()
def sonar = new org.devops.Sonar()
def mygit = new org.devops.Gitlab()


//使用git 参数需要格式化
env.branchName = "${env.branchName}" - "origin/"
println(env.branchName)

pipeline {
    agent {label "build"}

    //跳过默认的代码检出功能
    options {
        skipDefaultCheckout true
    }    
    

    stages{
        stage("CheckOut"){
            steps{
                script{
                    checkout.CheckOut()
                    
                    //获取commitID
                    env.commitID = checkout.GetCommitID()
                    println("commitID: ${env.commitID}")

                    // Jenkins构建显示信息
                    currentBuild.displayName = "第${BUILD_NUMBER}次构建-${env.commitID}"
                    currentBuild.description = "构建分支名称：${env.branchName}"
                    //currentBuild.description = "Trigger by user jenkins \n branch: ${env.branchName}"
                }
            }
        }

        stage("Build"){
            steps{
                script{
                    build.Build()
                }
            }

        }        

        stage("CodeScan"){
            // 是否跳过代码扫描？
            when {
                environment name: 'skipSonar', value: 'false'
            }
            
            steps{
                script{
                    sonar.SonarScannerByPlugin()        

                }
            }
        }

        stage("ImageBuild"){
            steps{
                script{
                    //PushArtifactByPlugin()
                    //PushArtifactByPluginPOM()

                    // init package info
                    appName = "${JOB_NAME}".split('_')[0] //devops6-maven-service_CI
                    repoName = appName.split('-')[0]   //devops6
  
                    imageName = "${repoName}/${appName}"
                    imageTag = "${env.branchName}-${env.commitID}"
                    env.fullImageName = "172.29.9.120/${imageName}:${imageTag}"
                    sh """
                        #登录镜像仓库
                        docker login -u admin -p Harbor12345 172.29.9.120

                        # 构建镜像
                        docker build -t ${env.fullImageName} .

                        # 上传镜像
                        docker push ${env.fullImageName}

                        # 删除镜像
                        sleep 2
                        docker rmi ${env.fullImageName}
                    """
                }
            }
    
        }
        
        stage("UpdateEnvFile"){
            steps{
                script {
                    // 更新部署文件

                    projectId = 11
                    fileName = "Deployment.yaml"  //模板文件
                    branchName = "main"

                    //下载模板文件
                    fileData = mygit.GetRepoFile(projectId,fileName,branchName)
                    sh "rm -fr ${fileName}"

                    //模板文件内容保存到本地
                    writeFile file: fileName , text: fileData
                    env.deployFile = fileName
                    //替换镜像
                    sh "sed -i 's#__IMAGE_NAME__#${env.fullImageName}#g' ${env.deployFile} "
                    sh "ls -l ; cat ${fileName}"

                    //创建/更新发布文件
                    newYaml = sh returnStdout: true, script: "cat ${env.deployFile}"
                    println(newYaml)
                   
                    //更新gitlab文件内容
                    base64Content = newYaml.bytes.encodeBase64().toString()
                    
                    appName = "${JOB_NAME}".split('_')[0] //devops6-npm-service
                    env.groupName = appName.split('-')[0]   //devops6
                    env.projectName = appName

                    // 会有并行问题，同时更新报错
                    try {
                        mygit.UpdateRepoFile(projectId,"${env.projectName}%2f${env.branchName}.yaml",base64Content, "main")
                    } catch(e){
                        mygit.CreateRepoFile(projectId,"${env.projectName}%2f${env.branchName}.yaml",base64Content, "main")
                    }
                }
            }
        }              

    }
}

k8scd.jenkinsfile文件：

@Library("devops06@main") _ 

def mygit = new org.devops.Gitlab()

//使用git 参数需要格式化
env.branchName = "${env.branchName}" - "origin/"
println(env.branchName)

pipeline {
    agent { label "build"}
    options {
        skipDefaultCheckout true
    }
    stages{
        stage("GetManifests"){
            steps{
                script{
                   
                   //下载发布文件
                    projectId = 11
                    env.deployFile = "${env.branchName}.yaml"  //版本分支RELEASE-2.1.1.yaml
                    env.appName = "${JOB_NAME}".split('_')[0] //devops6-maven-service
                    filePath = "${env.appName}%2f${env.deployFile}" //devops6-npm-service/RELEASE-2.1.1.yaml
                    branchName = "main"
                    fileData = mygit.GetRepoFile(projectId,filePath,branchName)
                    sh "rm -fr ${env.deployFile}"
                    writeFile file: env.deployFile , text: fileData
                    sh "ls -l ; cat ${env.deployFile}"
                }
            }
        }

        stage("Deploy"){
            steps{
                script{
                    env.namespace = "${env.appName}".split('-')[0]   //devops6

                    sh """
                        ## 发布应用
                        kubectl apply -f ${env.deployFile} -n ${env.namespace}

                    """

                    // 获取应用状态
                    5.times{
                        sh "sleep 2; kubectl -n ${env.namespace} get pod | grep ${env.appName}"
                    }
                }
            }
        }

        stage("RollBack"){
            input {
                message "是否进行回滚"
                ok "提交"
                submitter ""
                parameters {
                    choice(choices: ['yes', 'no'], name: 'opts')
                }
            }
            steps{
                script{
                    switch("${opts}") {
                        case "yes":
                            sh "kubectl rollout undo deployment/${env.appName} -n ${env.namespace} "
                            break
                        case "no":
                            break
                    }
                }
            }
        }
    }
}

2、Helm CI/CD流水线

tstmp_20230709102436

环境准备

删除devops6命名空间

[root@Devops6 ~]#kubectl delete ns devops6

集群安装helm

[root@Devops6 ~]#tar xf helm-v3.7.2-linux-amd64.tar.gz
[root@Devops6 ~]#cd linux-amd64/
[root@Devops6 linux-amd64]#cp helm /usr/bin/
[root@Devops6 linux-amd64]#chmod +x  /usr/bin/helm
[root@Devops6 linux-amd64]#helm version
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
version.BuildInfo{Version:"v3.7.2", GitCommit:"663a896f4a815053445eec4153677ddc24a0a361", GitTreeState:"clean", GoVersion:"go1.16.10"}
[root@Devops6 linux-amd64]#helm repo add stable http://mirror.azure.cn/kubernetes/charts/
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
"stable" has been added to your repositories
[root@Devops6 linux-amd64]# helm repo list
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
NAME  	URL                                      
stable	http://mirror.azure.cn/kubernetes/charts/
[root@Devops6 linux-amd64]#echo "source <(helm completion bash)" >> ~/.bashrc
[root@Devops6 linux-amd64]#source ~/.bashrc
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
[root@Devops6 linux-amd64]#helm list 
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
NAME	NAMESPACE	REVISION	UPDATED	STATUS	CHART	APP VERSION
[root@Devops6 linux-amd64]#

具体安装文档参考链接：

https://blog.csdn.net/weixin_39246554/article/details/123955289?ops_request_misc=&request_id=5c6a4510d72e40eabbcc39450945e6e4&biz_id=&utm_medium=distribute.pc_search_result.none-task-blog-2~blog~koosearch~default-2-123955289-null-null.268^v1^control&utm_term=helm&spm=1018.2226.3001.4450

创建helm仓库

devops6-helm-repo

helm手动安装包

生成helm chart

[root@Devops6 ~]#helm create devops6-npm-service
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
Creating devops6-npm-service
[root@Devops6 ~]#cd devops6-npm-service/
[root@Devops6 devops6-npm-service]#ls
charts  Chart.yaml  templates  values.yaml

编辑values.yaml

修改如下2处地方：

[root@Devops6 devops6-npm-service]#vim values.yaml
image:
  repository: 172.29.9.120/devops6/devops6-npm-service
  pullPolicy: IfNotPresent
  # Overrides the image tag whose default is the chart appVersion.
  tag: "RELEASE-2.1.1-8a398eee"

……
ingress:
  enabled: true
  className: ""
  annotations:
    kubernetes.io/ingress.class: nginx
    # kubernetes.io/tls-acme: "true"
  hosts:
    - host: devops.test.com
      paths:
        - path: /
          pathType: Prefix

生成部署文件

[root@Devops6 devops6-npm-service]#helm template --output-dir manifests .
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
wrote manifests/devops6-npm-service/templates/serviceaccount.yaml
wrote manifests/devops6-npm-service/templates/service.yaml
wrote manifests/devops6-npm-service/templates/deployment.yaml
wrote manifests/devops6-npm-service/templates/ingress.yaml
wrote manifests/devops6-npm-service/templates/tests/test-connection.yaml

部署

[root@Devops6 devops6-npm-service]#helm install devops6-npm-service . -ndevops6 --create-namespace 
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
NAME: devops6-npm-service
LAST DEPLOYED: Sat Jul  8 10:57:40 2023
NAMESPACE: devops6
STATUS: deployed
REVISION: 1
NOTES:
1. Get the application URL by running these commands:
  http://devops.test.com/

验证

[root@Devops6 devops6-npm-service]#kubectl get po -ndevops6
NAME                                   READY   STATUS    RESTARTS   AGE
devops6-npm-service-7bcb6c49b5-rls6f   1/1     Running   0          28s

[root@Devops6 devops6-npm-service]#watch -n 1 "curl -s devops.test.com"
Every 1.0s: curl -s devops.test.com             Sat Jul  8 10:58:46 2023                                                                                             
<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width,initial-scale=1.0">
    <title>vuedemo</title>
  </head>
  <body>
    <div id="app">
      <h1>VERSION: RELEASE-2.1.1</h1>
    </div>
    <!-- built files will be auto injected -->
  </body>
</html>

部署成功。

将次helm chart推送到刚才创建的helm仓库

#先删除manifests目录
[root@Devops6 devops6-npm-service]#pwd
/root/devops6-npm-service
[root@Devops6 devops6-npm-service]#ls
charts  Chart.yaml  manifests  templates  values.yaml
[root@Devops6 devops6-npm-service]#rm -rf manifests/

#推送 （注意：这次试验这里是master分支）
cd existing_folder
git init
git remote add origin http://172.29.9.101:8076/devops6/devops6-helm-repo.git
git add .
git commit -m "Initial commit"
git push -u origin master

harbor上开启helm chart

默认harbor没有helm chart功能，需要另外配置下才行

配置

开启helm charts
./install.sh  --with-chartmuseum
helm repo add devops6repo http://172.29.9.120/chartrepo/devops6/  --username=admin --password=Harbor12345

配置完后，harbor界面就出现了helm chart选项了：

添加过程：

[root@Devops6 devops6-npm-service]#helm repo list
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
NAME  	URL                                      
stable	http://mirror.azure.cn/kubernetes/charts/
[root@Devops6 devops6-npm-service]#helm repo add devops6repo http://172.29.9.120/chartrepo/devops6/  --username=admin --password=Harbor12345
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
"devops6repo" has been added to your repositories
[root@Devops6 devops6-npm-service]#helm repo list
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
NAME       	URL                                      
stable     	http://mirror.azure.cn/kubernetes/charts/
devops6repo	http://172.29.9.120/chartrepo/devops6/   

helm安装cm-push软件

CI pipeline

pipeline代码

创建CI 流水线

以devops6-npm-service_K8SCI为基础新创建devops6-npm-service_HELMCI项目

修改为helmci.jenkinsfile。

最终

CI 测试

CD pipeline

pipeline代码

创建CD流水线

以devops6-npm-service_K8SCD为基础新创建devops6-npm-service_HELMCD项目

修改为helmcd.jenkinsfile

修改选线参数：

最终：

CD测试

完整代码

完整代码如下：

链接：https://pan.baidu.com/s/1XFtZ0epIwgVwu0jzQHyJkA?pwd=0820 提取码：0820

2023.7.9-day9-k8s-ci-cd （kubectl和helm ci-cd）

CI/CD共享库源码前端项目（Dockerfile） K8s清单文件代码仓库 helm chart仓库

环境：

gitlab-ce:15.0.3-ce.0
jenkins:2.346.3-2-lts-jdk11
sonarqube:9.9.0-community
harbor v2.6.2  

helmci.jenkinsfile文件

@Library("devops06@main") _ 

//import  src/org/devops/Build.groovy
def build = new org.devops.Build()
def sonar = new org.devops.Sonar()
def checkouts = new org.devops.CheckOut()
def mygit = new org.devops.Gitlab()

//使用git 参数需要格式化
env.branchName = "${env.branchName}" - "origin/"
println(env.branchName)

pipeline {
    agent { label "build"}
    options {
        skipDefaultCheckout true
    }
    stages{
        stage("CheckOut"){
            steps{
                script{
                   checkouts.CheckOut()
                }
            }
        }
        stage("Build"){
            steps{
                script{
                    build.Build()
                }
            }
        }

        stage("CodeScan"){
            steps{
                script{
                    sonar.SonarScannerByPlugin()
                }
            }
        }
        stage("ImageBuild"){
            steps{
                script{
                    //PushArtifactByPlugin()
                    //PushArtifactByPluginPOM()
                    // init package info
                    appName = "${JOB_NAME}".split('_')[0] //devops6-maven-service
                    repoName = appName.split('-')[0]   //devops6
                    
                    //commitID
                    commitID = checkouts.GetCommitID()
                    println(commitID)

                    // Jenkins构建显示信息
                    currentBuild.displayName = "第${BUILD_NUMBER}次构建-${commitID}"
                    currentBuild.description = "构建分支名称：${env.branchName}"

                    imageName = "${repoName}/${appName}"
                    env.imageTag = "${env.branchName}-${commitID}"
                    env.fullImageName = "172.29.9.120/${imageName}:${env.imageTag}"
                    sh """
                        #登录镜像仓库
                        docker login -u admin -p Harbor12345 172.29.9.120

                        # 构建镜像
                        docker build -t ${env.fullImageName} .

                        # 上传镜像
                        docker push ${env.fullImageName}

                        # 删除镜像
                        sleep 2
                        docker rmi ${env.fullImageName}
                    """ 
                }
            }
        }
        stage("UpdateEnvFile"){
            steps{
                script {
                    // 更新部署文件

                    projectId = 12 //helm repo id
                    fileName = "values.yaml"  //模板文件
                    branchName = "master"
                    //下载模板文件
                    fileData = mygit.GetRepoFile(projectId,fileName,branchName)
                    sh "rm -fr ${fileName}"

                    //修改镜像tag
                    yamlData = readYaml text: fileData
                    yamlData.image.tag = "${env.imageTag}"

                    //模板文件内容保存到本地
                    writeYaml file: "${fileName}" , data: yamlData

                
                    //创建/更新发布文件
                    newYaml = sh returnStdout: true, script: "cat ${fileName}"
                    println(newYaml)
                    //更新gitlab文件内容
                    base64Content = newYaml.bytes.encodeBase64().toString()
                    

                    // 会有并行问题，同时更新报错
                    try {
                        mygit.UpdateRepoFile(projectId,"${fileName}",base64Content, "master")
                    } catch(e){
                        mygit.CreateRepoFile(projectId,"${fileName}",base64Content, "master")
                    }
                }
            }
        }
        stage("HelmPackage"){
            steps{
                script{
                    // helm pakcage & push harbor
                    appName = "${JOB_NAME}".split('_')[0]
                    sh "pwd && ls -l"
                    sh "mkdir -p ${appName} && cd ${appName}"
                    ws("${workspace}/${appName}"){
                        checkout([$class: 'GitSCM', branches: [[name: '*/master']], 
                            extensions: [], 
                            userRemoteConfigs: [[credentialsId: 'gitlab-root', 
                            url: 'http://172.29.9.101:8076/devops6/devops6-helm-repo.git']]])

                        sh "ls -l"
                    }
                    //helm package
                    chartVersion = "${env.branchName}".split("-")[-1]  //1.1.1
                    sh """ 
                        helm package ${appName} --version ${chartVersion}
                        helm-cm-push ${appName}-${chartVersion}.tgz devops6repo
                    """

                }
            }
        }
    }
}

helmcd.jenkinsfile文件

@Library("devops06@main") _ 

def checkout = new org.devops.CheckOut()
def build = new org.devops.Build()

//使用git 参数需要格式化
env.branchName = "${env.branchName}" - "origin/"
println(env.branchName)

//commitID
env.commitID = checkouts.GetCommitID()
println(commitID)

// Jenkins构建显示信息
currentBuild.displayName = "第${BUILD_NUMBER}次构建-${env.commitID}"
currentBuild.description = "构建分支名称：${env.branchName}"

pipeline {
    agent { label "build"}
    options {
        skipDefaultCheckout true
    }
    stages{
        stage("GetHelmChart"){
            steps{
                script{
                    //下载helm chart
                    env.chartVersion = "${env.branchName}".split("-")[-1]
                    env.appName = "${JOB_NAME}".split('_')[0]
                    sh """
                        helm repo update devops6repo
                        helm pull devops6repo/${env.appName} --version ${env.chartVersion}
                    """
                }
            }
        }

        stage("Deploy"){
            steps{
                script{
                    env.namespace = "${env.appName}".split('-')[0]   //devops6

                    sh """
                        ## 发布应用
                        helm upgrade --install --create-namespace  "${env.appName}" ./"${env.appName}"-${env.chartVersion}.tgz -n ${env.namespace}
                        helm history "${env.appName}"  -n ${env.namespace}
                    """

                    // 获取应用状态
                    5.times{
                        sh "sleep 2; kubectl -n ${env.namespace} get pod | grep ${env.appName}"
                    }

                    //收集history
                    env.revision = sh returnStdout: true, 
                                      script: """helm history ${env.appName} -n ${env.namespace} | grep -v 'REVISION' | awk '{print \$1}' """
                    println("${env.revision}")
                    println("${env.revision.split('\n').toString()}")
                    env.REVISION = "${env.revision.split('\n').toString()}"
                    println("${env.REVISION}")
                }
            }
        }


        stage("RollOut"){
            input {
                message "是否进行回滚"
                ok "提交"
                submitter ""
                parameters {
                    choice(choices: ['yes', 'no'], name: 'opts')
                }
            }

            steps{
                script{

                   switch("${opts}") {
                        case "yes":
                        def result = input  message: "选择回滚版本?", 
                            parameters: [choice(choices: env.REVISION, name: 'rversion')]

                        println("${result}")
                        sh "helm rollback ${env.appName} ${result} -n ${env.namespace} "
                        break

                        case "no":
                        break
                    }
                }
            }
        }
    }
}

关于我

我的博客主旨：

排版美观，语言精炼；
文档即手册，步骤明细，拒绝埋坑，提供源码；
本人实战文档都是亲测成功的，各位小伙伴在实际操作过程中如有什么疑问，可随时联系本人帮您解决问题，让我们一起进步！

🍀 微信二维码 x2675263825 （舍得）， qq：2675263825。

🍀 微信公众号《云原生架构师实战》

🍀 语雀

https://www.yuque.com/xyy-onlyone

🍀 csdn https://blog.csdn.net/weixin_39246554?spm=1010.2135.3001.5421

🍀 知乎 https://www.zhihu.com/people/foryouone

最后

好了，关于本次就到这里了，感谢大家阅读，最后祝大家生活快乐，每天都过的有意义哦，我们下期见！