kubernetes：无法使用具有 RW 访问权限的持久卷部署 jenkins 图像
kubernetes: can't deploy jenkins images with persistent volume with RW access
使用 kubernetes，我正在尝试部署 jenkins 映像和映射到 NFS 共享的持久卷（安装在我所有的工作人员上）
- 所以，这是我对我的工人的分享：
[root@pp-tmp-test24 /opt]# df -Th /opt/jenkins.persistent
Filesystem Type Size Used Avail Use% Mounted on
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP nfs4 10G 9.5M 10G 1% /opt/jenkins.persistent
- 还有我在这个分享上的数据
[root@pp-tmp-test24 /opt/jenkins.persistent]# ls -l
total 0
-rwxr-xr-x. 1 root root 0 Oct 2 11:53 newfile
[root@pp-tmp-test24 /opt/jenkins.persistent]# cat newfile
hello
- 这是我要部署的 yaml 文件
我的 PersistentVolume yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-pv-nfs
labels:
type: type-nfs
spec:
storageClassName: class-nfs
capacity:
storage: 10Gi
volumeMode: Filesystem
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Recycle
hostPath:
path: /opt/jenkins.persistent
我的 PersistentVolumeClaim yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: jenkins-pvc-nfs
namespace: ns-jenkins
spec:
storageClassName: class-nfs
volumeMode: Filesystem
accessModes:
- ReadWriteMany
resources:
requests:
storage: 10Gi
selector:
matchLabels:
type: type-nfs
还有我的部署
apiVersion: apps/v1
kind: Deployment
metadata:
name: jenkins
namespace: ns-jenkins
spec:
replicas: 1
selector:
matchLabels:
app: jenkins
template:
metadata:
labels:
app: jenkins
spec:
containers:
- image: jenkins
#- image: httpd:latest
name: jenkins
ports:
- containerPort: 8080
protocol: TCP
name: jenkins-web
volumeMounts:
- name: jenkins-persistent-storage
mountPath: /var/foo
volumes:
- name: jenkins-persistent-storage
persistentVolumeClaim:
claimName: jenkins-pvc-nfs
- 执行
kubectl create -f 命令后，一切正常：
# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
jenkins-pv-nfs 10Gi RWX Recycle Bound ns-jenkins/jenkins-pvc-nfs class-nfs 37s
# kubectl get pvc -A
NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ns-jenkins jenkins-pvc-nfs Bound jenkins-pv-nfs 10Gi RWX class-nfs 35s
# kubectl get pods -A |grep jenkins
ns-jenkins jenkins-5bdb8678c-x6vht 1/1 Running 0 14s
# kubectl describe pod jenkins-5bdb8678c-x6vht -n ns-jenkins
Name: jenkins-5bdb8678c-x6vht
Namespace: ns-jenkins
Priority: 0
Node: pp-tmp-test25.mydomain/172.31.68.225
Start Time: Wed, 02 Oct 2019 11:48:23 +0200
Labels: app=jenkins
pod-template-hash=5bdb8678c
Annotations: <none>
Status: Running
IP: 10.244.5.47
Controlled By: ReplicaSet/jenkins-5bdb8678c
Containers:
jenkins:
Container ID: docker://8a3e4871ed64b371818bac59e24d6912e5d2b13c8962c1639d36797fbce8082e
Image: jenkins
Image ID: docker-pullable://docker.io/jenkins@sha256:eeb4850eb65f2d92500e421b430ed1ec58a7ac909e91f518926e02473904f668
Port: 8080/TCP
Host Port: 0/TCP
State: Running
Started: Wed, 02 Oct 2019 11:48:26 +0200
Ready: True
Restart Count: 0
Environment: <none>
Mounts:
/var/foo from jenkins-persistent-storage (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-dz6cd (ro)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
jenkins-persistent-storage:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: jenkins-pvc-nfs
ReadOnly: false
default-token-dz6cd:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-dz6cd
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 39s default-scheduler Successfully assigned ns-jenkins/jenkins-5bdb8678c-x6vht to pp-tmp-test25.mydomain
Normal Pulling 38s kubelet, pp-tmp-test25.mydomain Pulling image "jenkins"
Normal Pulled 36s kubelet, pp-tmp-test25.mydomain Successfully pulled image "jenkins"
Normal Created 36s kubelet, pp-tmp-test25.mydomain Created container jenkins
Normal Started 36s kubelet, pp-tmp-test25.mydomain Started container jenkins
- 在我的工人身上，这是我的容器
# docker ps |grep jenkins
8a3e4871ed64 docker.io/jenkins@sha256:eeb4850eb65f2d92500e421b430ed1ec58a7ac909e91f518926e02473904f668 "/bin/tini -- /usr..." 2 minutes ago Up 2 minutes k8s_jenkins_jenkins-5bdb8678c-x6vht_ns-jenkins_64b66dae-a1da-4d90-83fd-ff433638dc9c_0
所以我在我的容器上启动了一个 shell，我可以在 /var/foo 上看到我的数据：
# docker exec -t -i 8a3e4871ed64 /bin/bash
jenkins@jenkins-5bdb8678c-x6vht:/$ df -h /var/foo
Filesystem Size Used Avail Use% Mounted on
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP 10G 9.5M 10G 1% /var/foo
jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ ls -lZ /var/foo -d
drwxr-xr-x. 2 root root system_u:object_r:nfs_t:s0 4096 Oct 2 10:06 /var/foo
jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ ls -lZ /var/foo
-rwxr-xr-x. 1 root root system_u:object_r:nfs_t:s0 12 Oct 2 10:05 newfile
jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ cat newfile
hello
我正尝试在我的 /var/foo/newfile 中写入数据，但权限被拒绝
jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ echo "world" >> newfile
bash: newfile: Permission denied
我的 /var/foo/ directory 也一样，我无法写入数据
jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ touch newfile2
touch: cannot touch 'newfile2': Permission denied
所以，我在我的部署 yaml 中尝试了另一个像 httpd:latest 这样的图像（在我的 yaml 定义中保持相同的名称）
[...]
containers:
#- image: jenkins
- image: httpd:latest
[...]
# docker ps |grep jenkins
fa562400405d docker.io/httpd@sha256:39d7d9a3ab93c0ad68ee7ea237722ed1b0016ff6974d80581022a53ec1e58797 "httpd-foreground" 50 seconds ago Up 48 seconds k8s_jenkins_jenkins-7894877f96-6dj85_ns-jenkins_540b12bd-69df-44d8-b3df-20a0a96cc851_0
在我的新容器中，这次我可以读写数据了：
root@jenkins-7894877f96-6dj85:/usr/local/apache2# df -h /var/foo
Filesystem Size Used Avail Use% Mounted on
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP 10G 9.6M 10G 1% /var/foo
root@jenkins-7894877f96-6dj85:/var/foo# ls -lZ
total 0
-rwxr-xr-x. 1 root root system_u:object_r:nfs_t:s0 12 Oct 2 10:05 newfile
-rw-r--r--. 1 root root system_u:object_r:nfs_t:s0 0 Oct 2 10:06 newfile2
root@jenkins-7894877f96-6dj85:/var/foo# ls -lZ /var/foo -d
drwxr-xr-x. 2 root root system_u:object_r:nfs_t:s0 4096 Oct 2 10:06 /var/foo
root@jenkins-7894877f96-6dj85:/var/foo# ls -l
total 0
-rwxr-xr-x. 1 root root 6 Oct 2 09:55 newfile
root@jenkins-7894877f96-6dj85:/var/foo# echo "world" >> newfile
root@jenkins-7894877f96-6dj85:/var/foo# touch newfile2
root@jenkins-7894877f96-6dj85:/var/foo# ls -l
total 0
-rwxr-xr-x. 1 root root 12 Oct 2 10:05 newfile
-rw-r--r--. 1 root root 0 Oct 2 10:06 newfile2
我做错了什么？ pb 是由于 jenkins 不允许 RW 访问的图像造成的吗？与具有持久卷的本地存储（在我的工作人员上）相同的 pb。
其他事情，也许是愚蠢的：对于我的 jenkins 映像，我想将 /var/jenkins_home 目录挂载到持久卷以保留 jenkins 的配置文件。但是如果我尝试挂载 /var/jenkins_home 而不是 /var/foo，pod 会崩溃（因为已经有数据存储在 /var/jenkins_home 中）。
谢谢大家的帮助！
我注意到您正在尝试以 jenkins 用户的身份在 jenkins-5bdb8678c-x6vht 上写入，而该用户在 root:root 目录中可能没有写入权限。
您可能想要更改该目录权限以匹配 jenkins 用户权限。
在写入文件之前，尝试使用 sudo 来验证是否是导致此问题的原因。
如果您未安装 sudo，则以 root 用户身份使用 --user 标志执行。所以它就像在其他写作有效的情况下一样。
docker exec -t -i -u root 8a3e4871ed64 /bin/bash
@Piotr Malec 谢谢。是的，我意识到：当我连接到我的容器时，jenkins 是默认用户：
docker exec -t -i 46d2497d440d /bin/bash
jenkins@jenkins-7bcdd5db57-8qgth:/$
因此，为了尝试，我已将工作人员对此 /opt/jenkins.persistent 的权限更改为 777，现在我在此安装上拥有 RW perm :
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP 10G 9.5M 10G 1% /var/foo
jenkins@jenkins-7bcdd5db57-8qgth:/$ cd /var
jenkins@jenkins-7bcdd5db57-8qgth:/$ ls -l
[...]
drwxrwxrwx. 2 root root 4096 Oct 4 13:41 foo
[...]
jenkins@jenkins-7bcdd5db57-8qgth:/$ cd /var/foo
jenkins@jenkins-7bcdd5db57-8qgth:/var/foo $ touch newfile
jenkins@jenkins-7bcdd5db57-8qgth:/var/foo $ ls
newfile
所以我在我的工作人员上添加了 jenkins 用户帐户，并在我的 /opt/jenkins.persistent 目录上设置了 chown jenkins:jenkins。现在，在我的容器中我有 RW perm :
jenkins@jenkins-7bcdd5db57-8qgth:/var$ ls -l
[...]
drwxr-xr-x. 2 jenkins jenkins 4096 Oct 4 13:53 foo
[...]
jenkins@jenkins-7bcdd5db57-8qgth:/var$ cd foo
jenkins@jenkins-7bcdd5db57-8qgth:/var/toto$ touch newfile2
jenkins@jenkins-7bcdd5db57-8qgth:/var/toto$ ls -l
-rw-r--r--. 1 jenkins jenkins 0 Oct 4 13:53 newfile2
使用 kubernetes，我正在尝试部署 jenkins 映像和映射到 NFS 共享的持久卷（安装在我所有的工作人员上）
- 所以，这是我对我的工人的分享：
[root@pp-tmp-test24 /opt]# df -Th /opt/jenkins.persistent
Filesystem Type Size Used Avail Use% Mounted on
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP nfs4 10G 9.5M 10G 1% /opt/jenkins.persistent
- 还有我在这个分享上的数据
[root@pp-tmp-test24 /opt/jenkins.persistent]# ls -l
total 0
-rwxr-xr-x. 1 root root 0 Oct 2 11:53 newfile
[root@pp-tmp-test24 /opt/jenkins.persistent]# cat newfile
hello
- 这是我要部署的 yaml 文件
我的 PersistentVolume yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-pv-nfs
labels:
type: type-nfs
spec:
storageClassName: class-nfs
capacity:
storage: 10Gi
volumeMode: Filesystem
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Recycle
hostPath:
path: /opt/jenkins.persistent
我的 PersistentVolumeClaim yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: jenkins-pvc-nfs
namespace: ns-jenkins
spec:
storageClassName: class-nfs
volumeMode: Filesystem
accessModes:
- ReadWriteMany
resources:
requests:
storage: 10Gi
selector:
matchLabels:
type: type-nfs
还有我的部署
apiVersion: apps/v1
kind: Deployment
metadata:
name: jenkins
namespace: ns-jenkins
spec:
replicas: 1
selector:
matchLabels:
app: jenkins
template:
metadata:
labels:
app: jenkins
spec:
containers:
- image: jenkins
#- image: httpd:latest
name: jenkins
ports:
- containerPort: 8080
protocol: TCP
name: jenkins-web
volumeMounts:
- name: jenkins-persistent-storage
mountPath: /var/foo
volumes:
- name: jenkins-persistent-storage
persistentVolumeClaim:
claimName: jenkins-pvc-nfs
- 执行
kubectl create -f命令后，一切正常：
# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
jenkins-pv-nfs 10Gi RWX Recycle Bound ns-jenkins/jenkins-pvc-nfs class-nfs 37s
# kubectl get pvc -A
NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
ns-jenkins jenkins-pvc-nfs Bound jenkins-pv-nfs 10Gi RWX class-nfs 35s
# kubectl get pods -A |grep jenkins
ns-jenkins jenkins-5bdb8678c-x6vht 1/1 Running 0 14s
# kubectl describe pod jenkins-5bdb8678c-x6vht -n ns-jenkins
Name: jenkins-5bdb8678c-x6vht
Namespace: ns-jenkins
Priority: 0
Node: pp-tmp-test25.mydomain/172.31.68.225
Start Time: Wed, 02 Oct 2019 11:48:23 +0200
Labels: app=jenkins
pod-template-hash=5bdb8678c
Annotations: <none>
Status: Running
IP: 10.244.5.47
Controlled By: ReplicaSet/jenkins-5bdb8678c
Containers:
jenkins:
Container ID: docker://8a3e4871ed64b371818bac59e24d6912e5d2b13c8962c1639d36797fbce8082e
Image: jenkins
Image ID: docker-pullable://docker.io/jenkins@sha256:eeb4850eb65f2d92500e421b430ed1ec58a7ac909e91f518926e02473904f668
Port: 8080/TCP
Host Port: 0/TCP
State: Running
Started: Wed, 02 Oct 2019 11:48:26 +0200
Ready: True
Restart Count: 0
Environment: <none>
Mounts:
/var/foo from jenkins-persistent-storage (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-dz6cd (ro)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
jenkins-persistent-storage:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: jenkins-pvc-nfs
ReadOnly: false
default-token-dz6cd:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-dz6cd
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 39s default-scheduler Successfully assigned ns-jenkins/jenkins-5bdb8678c-x6vht to pp-tmp-test25.mydomain
Normal Pulling 38s kubelet, pp-tmp-test25.mydomain Pulling image "jenkins"
Normal Pulled 36s kubelet, pp-tmp-test25.mydomain Successfully pulled image "jenkins"
Normal Created 36s kubelet, pp-tmp-test25.mydomain Created container jenkins
Normal Started 36s kubelet, pp-tmp-test25.mydomain Started container jenkins
- 在我的工人身上，这是我的容器
# docker ps |grep jenkins
8a3e4871ed64 docker.io/jenkins@sha256:eeb4850eb65f2d92500e421b430ed1ec58a7ac909e91f518926e02473904f668 "/bin/tini -- /usr..." 2 minutes ago Up 2 minutes k8s_jenkins_jenkins-5bdb8678c-x6vht_ns-jenkins_64b66dae-a1da-4d90-83fd-ff433638dc9c_0
所以我在我的容器上启动了一个 shell，我可以在 /var/foo 上看到我的数据：
# docker exec -t -i 8a3e4871ed64 /bin/bash
jenkins@jenkins-5bdb8678c-x6vht:/$ df -h /var/foo
Filesystem Size Used Avail Use% Mounted on
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP 10G 9.5M 10G 1% /var/foo
jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ ls -lZ /var/foo -d
drwxr-xr-x. 2 root root system_u:object_r:nfs_t:s0 4096 Oct 2 10:06 /var/foo
jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ ls -lZ /var/foo
-rwxr-xr-x. 1 root root system_u:object_r:nfs_t:s0 12 Oct 2 10:05 newfile
jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ cat newfile
hello
我正尝试在我的 /var/foo/newfile 中写入数据，但权限被拒绝
jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ echo "world" >> newfile
bash: newfile: Permission denied
我的 /var/foo/ directory 也一样，我无法写入数据
jenkins@jenkins-5bdb8678c-x6vht:/var/foo$ touch newfile2
touch: cannot touch 'newfile2': Permission denied
所以，我在我的部署 yaml 中尝试了另一个像 httpd:latest 这样的图像（在我的 yaml 定义中保持相同的名称）
[...]
containers:
#- image: jenkins
- image: httpd:latest
[...]
# docker ps |grep jenkins
fa562400405d docker.io/httpd@sha256:39d7d9a3ab93c0ad68ee7ea237722ed1b0016ff6974d80581022a53ec1e58797 "httpd-foreground" 50 seconds ago Up 48 seconds k8s_jenkins_jenkins-7894877f96-6dj85_ns-jenkins_540b12bd-69df-44d8-b3df-20a0a96cc851_0
在我的新容器中，这次我可以读写数据了：
root@jenkins-7894877f96-6dj85:/usr/local/apache2# df -h /var/foo
Filesystem Size Used Avail Use% Mounted on
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP 10G 9.6M 10G 1% /var/foo
root@jenkins-7894877f96-6dj85:/var/foo# ls -lZ
total 0
-rwxr-xr-x. 1 root root system_u:object_r:nfs_t:s0 12 Oct 2 10:05 newfile
-rw-r--r--. 1 root root system_u:object_r:nfs_t:s0 0 Oct 2 10:06 newfile2
root@jenkins-7894877f96-6dj85:/var/foo# ls -lZ /var/foo -d
drwxr-xr-x. 2 root root system_u:object_r:nfs_t:s0 4096 Oct 2 10:06 /var/foo
root@jenkins-7894877f96-6dj85:/var/foo# ls -l
total 0
-rwxr-xr-x. 1 root root 6 Oct 2 09:55 newfile
root@jenkins-7894877f96-6dj85:/var/foo# echo "world" >> newfile
root@jenkins-7894877f96-6dj85:/var/foo# touch newfile2
root@jenkins-7894877f96-6dj85:/var/foo# ls -l
total 0
-rwxr-xr-x. 1 root root 12 Oct 2 10:05 newfile
-rw-r--r--. 1 root root 0 Oct 2 10:06 newfile2
我做错了什么？ pb 是由于 jenkins 不允许 RW 访问的图像造成的吗？与具有持久卷的本地存储（在我的工作人员上）相同的 pb。
其他事情，也许是愚蠢的：对于我的 jenkins 映像，我想将 /var/jenkins_home 目录挂载到持久卷以保留 jenkins 的配置文件。但是如果我尝试挂载 /var/jenkins_home 而不是 /var/foo，pod 会崩溃（因为已经有数据存储在 /var/jenkins_home 中）。
谢谢大家的帮助！
我注意到您正在尝试以 jenkins 用户的身份在 jenkins-5bdb8678c-x6vht 上写入，而该用户在 root:root 目录中可能没有写入权限。
您可能想要更改该目录权限以匹配 jenkins 用户权限。
在写入文件之前，尝试使用 sudo 来验证是否是导致此问题的原因。
如果您未安装 sudo，则以 root 用户身份使用 --user 标志执行。所以它就像在其他写作有效的情况下一样。
docker exec -t -i -u root 8a3e4871ed64 /bin/bash
@Piotr Malec 谢谢。是的，我意识到：当我连接到我的容器时，jenkins 是默认用户：
docker exec -t -i 46d2497d440d /bin/bash
jenkins@jenkins-7bcdd5db57-8qgth:/$
因此，为了尝试，我已将工作人员对此 /opt/jenkins.persistent 的权限更改为 777，现在我在此安装上拥有 RW perm :
xxx.xxx.xxx.xxx:/VR_C_CS003_NFS_KUBERNETESPV_TMP_PP 10G 9.5M 10G 1% /var/foo
jenkins@jenkins-7bcdd5db57-8qgth:/$ cd /var
jenkins@jenkins-7bcdd5db57-8qgth:/$ ls -l
[...]
drwxrwxrwx. 2 root root 4096 Oct 4 13:41 foo
[...]
jenkins@jenkins-7bcdd5db57-8qgth:/$ cd /var/foo
jenkins@jenkins-7bcdd5db57-8qgth:/var/foo $ touch newfile
jenkins@jenkins-7bcdd5db57-8qgth:/var/foo $ ls
newfile
所以我在我的工作人员上添加了 jenkins 用户帐户，并在我的 /opt/jenkins.persistent 目录上设置了 chown jenkins:jenkins。现在，在我的容器中我有 RW perm :
jenkins@jenkins-7bcdd5db57-8qgth:/var$ ls -l
[...]
drwxr-xr-x. 2 jenkins jenkins 4096 Oct 4 13:53 foo
[...]
jenkins@jenkins-7bcdd5db57-8qgth:/var$ cd foo
jenkins@jenkins-7bcdd5db57-8qgth:/var/toto$ touch newfile2
jenkins@jenkins-7bcdd5db57-8qgth:/var/toto$ ls -l
-rw-r--r--. 1 jenkins jenkins 0 Oct 4 13:53 newfile2