Jenkins-ci
提到基于 Kubernetes 的 CI/CD，可以使用的工具有很多，比如 Jenkins、gitlab-runner 等，我们这里会使用大家最为熟悉的 Jenkins 来做 CI/CD 的工具。环境:
- 阿里云镜像仓库
- github代码仓库
- Kubernetes-v1.25.12
- Jenkins
# 1. 部署jenkins
[root@ci ~]# cat jenkins.yaml
apiVersion: v1
kind: Namespace
metadata:
name: kube-ops
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-local
labels:
app: jenkins
spec:
accessModes:
- ReadWriteOnce
capacity:
storage: 5Gi
storageClassName: local-storage
local:
path: /data/k8s/jenkins
persistentVolumeReclaimPolicy: Retain
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- node2
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: jenkins-pvc
namespace: kube-ops
spec:
storageClassName: local-storage
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: jenkins
namespace: kube-ops
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: jenkins
rules:
- apiGroups: ["extensions", "apps"]
resources: ["deployments", "ingresses"]
verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
- apiGroups: [""]
resources: ["services"]
verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]
- apiGroups: [""]
resources: ["pods/log", "events"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: jenkins
namespace: kube-ops
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: jenkins
subjects:
- kind: ServiceAccount
name: jenkins
namespace: kube-ops
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: jenkins
namespace: kube-ops
spec:
selector:
matchLabels:
app: jenkins
template:
metadata:
labels:
app: jenkins
spec:
serviceAccount: jenkins
initContainers:
- name: fix-permissions
image: registry.cn-hangzhou.aliyuncs.com/s-ops/busybox:1.35.0
command: ["sh", "-c", "chown -R 1000:1000 /var/jenkins_home"]
securityContext:
privileged: true
volumeMounts:
- name: jenkinshome
mountPath: /var/jenkins_home
containers:
- name: jenkins
image: registry.cn-hangzhou.aliyuncs.com/s-ops/jenkins:lts-jdk11
imagePullPolicy: IfNotPresent
env:
- name: JAVA_OPTS
value: -Dhudson.model.DownloadService.noSignatureCheck=true -Duser.timezone=Asia/Shanghai
ports:
- containerPort: 8080
name: web
protocol: TCP
- containerPort: 50000
name: agent
protocol: TCP
readinessProbe:
httpGet:
path: /login
port: 8080
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12
volumeMounts:
- name: jenkinshome
mountPath: /var/jenkins_home
volumes:
- name: jenkinshome
persistentVolumeClaim:
claimName: jenkins-pvc
---
apiVersion: v1
kind: Service
metadata:
name: jenkins
namespace: kube-ops
labels:
app: jenkins
spec:
selector:
app: jenkins
ports:
- name: web
port: 8080
targetPort: web
- name: agent
port: 50000
targetPort: agent
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: jenkins
namespace: kube-ops
spec:
ingressClassName: nginx
rules:
- host: jenkins.tbchip.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: jenkins
port:
name: web
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
# 2. 获取密码登陆
[root@ci ~]# kubectl logs -f -n kube-ops jenkins-7d64f47476-cl8ck
*************************************************************
*************************************************************
*************************************************************
Jenkins initial setup is required. An admin user has been created and a password generated.
Please use the following password to proceed to installation:
6cc33e046c8b4d27b7792c0cffbf31b2
This may also be found at: /var/jenkins_home/secrets/initialAdminPassword
*************************************************************
*************************************************************
*************************************************************
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# 3. 登陆jenkins web
# 3.1 选择插件
# 3.2 勾选无,在勾选Languages
# 3.3 点击安装,跟着指引登陆web页面.进入http://jenkins.tbchip.com/user/admin/configure,修改默认的admin的密码和时区
# 3.4 安装插件,http://jenkins.tbchip.com/manage/pluginManager/available
Git
Lock
Pipeline
timestamp
ThinBackup
Kubernetes
Pipeline View
Active Choices
Build Name and Description Setter
1
2
3
4
5
6
7
8
9
2
3
4
5
6
7
8
9
# 4. 配置动态slave节点
# 4.1 Kubernetes 的方式来创建动态节点,http://jenkins.tbchip.com/manage/cloud/new
# 4.2 配置连接 Kubernetes APIServer 的地址，由于 Jenkins 运行在 Kubernetes 集群中，所以可以使用 Service 的 DNS 形式进行连接 https://kubernetes.default.svc.cluster.local,命名空间这里填 kube-ops，然后点击 连接测试,出现k8s版本则表示链接正常:
# 4.3 下方的 Jenkins URL 地址为 http://jenkins.kube-ops.svc.cluster.local:8080，根据上面创建的 jenkins 的服务名填写，包括下面的 Jenkins 通道，默认是 50000 端口地址为jenkins.kube-ops.svc.cluster.local:50000,点击save:
# 5. 凭据配置
# 5.1 镜像仓库的用户名和密码信息则需要通过凭据来进行添加，进入 http://jenkins.tbchip.com/credentials/store/system/domain/_/ 页面添加凭据，选择用户名和密码类型的,按序填写镜像仓库的账号密码:
# 5.2 ssh密钥配置,http://jenkins.tbchip.com/manage/credentials/store/system/domain/_/newCredentials页面添加凭据,选择如下:
# 6. 实践
# 6.1 Jenkins基于 containerd的K8S集群进行多架构镜像编译:
➜ ci-demo git:(main) tree .
.
├── Dockerfile
├── Jenkinsfile
├── README.md
├── ci
│ └── jnlp.yaml
├── go.mod
├── go.sum
├── helm
│ ├── Chart.yaml
│ ├── templates
│ │ ├── deployment.yaml
│ │ └── service.yaml
│ └── values.yaml
├── main.go
└── values.tpl
4 directories, 12 files
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# 6.2 Jenkinsfile
// 使用split函数以斜杠为分隔符拆分字符串，并提取最后一个元素
def COMMITID = ""
def TIMESTAMP = ""
pipeline {
agent {
kubernetes {
label "jnlp-slave-${UUID.randomUUID().toString().substring(0, 8)}"
yamlFile "ci/jnlp.yaml"
}
}
environment {
DOCKER_REGISTRY = "registry.cn-hangzhou.aliyuncs.com"
REGISTRY_NAMEPSACE = "gitops-demo"
IMAGE = "${DOCKER_REGISTRY}/${REGISTRY_NAMEPSACE}"
}
options {
//保持构建15天 最大保持构建的30个 发布包保留15天
buildDiscarder logRotator(artifactDaysToKeepStr: '15', artifactNumToKeepStr: '', daysToKeepStr: '15', numToKeepStr: '30')
//时间模块
timestamps()
//超时时间
timeout(time:60, unit:'MINUTES')
}
stages {
stage('commit'){
steps{
script{
COMMITID = sh(script: "git rev-parse --short HEAD", returnStdout: true).trim()
TIMESTAMP = sh(script: "date +%Y%m%d%H%M-%S", returnStdout: true).trim()
env.ImageTag = "${BUILD_ID}-${TIMESTAMP}-${COMMITID}"
env.AppName = env.JOB_NAME.split('/').last().toLowerCase()
}
}
}
stage('build image') {
steps {
container('docker') {
withCredentials([[$class: 'UsernamePasswordMultiBinding',
credentialsId: 'docker-auth',
usernameVariable: 'DOCKER_USER',
passwordVariable: 'DOCKER_PASSWORD']]) {
script {
sh """
echo "开启多架构编译"
docker buildx create --name mybuilder --use --driver docker-container --driver-opt image=registry.cn-hangzhou.aliyuncs.com/s-ops/buildkit:buildx-stable-1
echo "登陆仓库"
docker login ${DOCKER_REGISTRY} -u ${DOCKER_USER} -p ${DOCKER_PASSWORD}
echo "构建/推送镜像"
docker buildx build --progress=plain --no-cache --platform=linux/amd64,linux/arm64 -f Dockerfile -t ${IMAGE}/${AppName}:${ImageTag} . --push
"""
}
}
}
}
}
stage('change ImageTag') {
steps {
container('tools'){
script{
sh """
envsubst < ./values.tpl > helm/values.yaml
cat helm/values.yaml
helm template --debug helm/ -f helm/values.yaml
"""
}
}
}
}
stage('push yaml') {
steps {
withCredentials([sshUserPrivateKey(credentialsId: 'github-ci', keyFileVariable: 'IDENTITY')]) {
script {
sh """
git config --global user.email "ci"
git config --global user.email "[email protected]"
git config core.sshCommand 'ssh -o StrictHostKeyChecking=no -i $IDENTITY'
git checkout main
git pull origin main
git add .
git commit -m "${AppName}-${ImageTag} " || true
git push origin main
"""
}
}
}
}
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
# 6.3 动态slave,ci/jnlp.yaml
apiVersion: v1
kind: Pod
metadata:
labels:
app: jenkins-slave
spec:
volumes:
- name: docker-socket
emptyDir: {}
- name: workspace-volume
emptyDir: {}
serviceAccount: jenkins
containers:
- name: jnlp
image: registry.cn-hangzhou.aliyuncs.com/s-ops/inbound-agent:latest
- name: tools
image: registry.cn-hangzhou.aliyuncs.com/s-ops/tools:latest
command:
- cat
tty: true
- name: docker
image: registry.cn-hangzhou.aliyuncs.com/s-ops/docker:latest
command:
- sleep
args:
- 99d
readinessProbe:
exec:
command: ["ls", "-S", "/var/run/docker.sock"]
initialDelaySeconds: 10
volumeMounts:
- name: docker-socket
mountPath: /var/run
- name: docker-daemon
image: registry.cn-hangzhou.aliyuncs.com/s-ops/docker:19.03.1-dind
securityContext:
privileged: true
volumeMounts:
- name: docker-socket
mountPath: /var/run
- name: workspace-volume
mountPath: /home/jenkins/agent
readOnly: false
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# 7. 问题
# 7.1 通过ssh拉取代码,密钥配置正确,但是任然无法正常拉取代码且报错,主要是因为主机密钥验证的问题,报错如图:
# 8. 解决方案
# 8.1 暴力解决方法,http://jenkins.tbchip.com/manage/configureSecurity/,将Git Host Key Verification Configuration中的Host Key Verification Strategy改为No verification:
# 8.2 通过在jenkins服务器上配置主机密钥验证,参考1 (opens new window)/参考2 (opens new window)
上次更新: 2025/04/25, 03:40:17