istio 得到 "RBAC: access denied" 甚至 servicerolebinding 检查被允许

istio getting "RBAC: access denied" even the servicerolebinding checked to be allowed

我一直在纠结istio...所以我在这里寻求专家的帮助!

背景

我正在尝试使用 dexmulti-tenency 部署我的 kubeflow 应用程序。 参考kubeflow offical document with the manifest file from github

这里是component/version信息列表

通过清单文件,我成功地将 kubeflow 部署到了我的集群上。这是我所做的。

这里是第3步和第4步的验证 检查已启用 RBAC 并为 kubeflow-userid

添加了 envoy 过滤器
[root@gke-client-tf leilichao]# k get clusterrbacconfigs -o yaml
apiVersion: v1
items:
- apiVersion: rbac.istio.io/v1alpha1
  kind: ClusterRbacConfig
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"rbac.istio.io/v1alpha1","kind":"ClusterRbacConfig","metadata":{"annotations":{},"name":"default"},"spec":{"mode":"ON"}}
    creationTimestamp: "2020-07-04T01:28:52Z"
    generation: 2
    name: default
    resourceVersion: "5986075"
    selfLink: /apis/rbac.istio.io/v1alpha1/clusterrbacconfigs/default
    uid: db70920e-f364-40ec-a93b-a3364f88650f
  spec:
    mode: "ON"
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""
[root@gke-client-tf leilichao]# k get envoyfilter -n istio-system -o yaml
apiVersion: v1
items:
- apiVersion: networking.istio.io/v1alpha3
  kind: EnvoyFilter
  metadata:
    annotations:
      kubectl.kubernetes.io/last-applied-configuration: |
        {"apiVersion":"networking.istio.io/v1alpha3","kind":"EnvoyFilter","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"oidc-authservice","app.kubernetes.io/instance":"oidc-authservice-v1.0.0","app.kubernetes.io/managed-by":"kfctl","app.kubernetes.io/name":"oidc-authservice","app.kubernetes.io/part-of":"kubeflow","app.kubernetes.io/version":"v1.0.0"},"name":"authn-filter","namespace":"istio-system"},"spec":{"filters":[{"filterConfig":{"httpService":{"authorizationRequest":{"allowedHeaders":{"patterns":[{"exact":"cookie"},{"exact":"X-Auth-Token"}]}},"authorizationResponse":{"allowedUpstreamHeaders":{"patterns":[{"exact":"kubeflow-userid"}]}},"serverUri":{"cluster":"outbound|8080||authservice.istio-system.svc.cluster.local","failureModeAllow":false,"timeout":"10s","uri":"http://authservice.istio-system.svc.cluster.local"}},"statusOnError":{"code":"GatewayTimeout"}},"filterName":"envoy.ext_authz","filterType":"HTTP","insertPosition":{"index":"FIRST"},"listenerMatch":{"listenerType":"GATEWAY"}}],"workloadLabels":{"istio":"ingressgateway"}}}
    creationTimestamp: "2020-07-04T01:40:43Z"
    generation: 1
    labels:
      app.kubernetes.io/component: oidc-authservice
      app.kubernetes.io/instance: oidc-authservice-v1.0.0
      app.kubernetes.io/managed-by: kfctl
      app.kubernetes.io/name: oidc-authservice
      app.kubernetes.io/part-of: kubeflow
      app.kubernetes.io/version: v1.0.0
    name: authn-filter
    namespace: istio-system
    resourceVersion: "4715289"
    selfLink: /apis/networking.istio.io/v1alpha3/namespaces/istio-system/envoyfilters/authn-filter
    uid: e599ba82-315a-4fc1-9a5d-e8e35d93ca26
  spec:
    filters:
    - filterConfig:
        httpService:
          authorizationRequest:
            allowedHeaders:
              patterns:
              - exact: cookie
              - exact: X-Auth-Token
          authorizationResponse:
            allowedUpstreamHeaders:
              patterns:
              - exact: kubeflow-userid
          serverUri:
            cluster: outbound|8080||authservice.istio-system.svc.cluster.local
            failureModeAllow: false
            timeout: 10s
            uri: http://authservice.istio-system.svc.cluster.local
        statusOnError:
          code: GatewayTimeout
      filterName: envoy.ext_authz
      filterType: HTTP
      insertPosition:
        index: FIRST
      listenerMatch:
        listenerType: GATEWAY
    workloadLabels:
      istio: ingressgateway
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

RBAC Issue问题分析

我完成部署后。我执行了以下功能测试:

RBAC 问题调查

在 kubeflow 上成功创建笔记本服务器并尝试连接笔记本服务器后,我收到“RBAC:访问被拒绝”错误。 我设法更新了 envoy 日志级别并得到以下日志。

[2020-08-06 13:32:43.290][26][debug][rbac] [external/envoy/source/extensions/filters/http/rbac/rbac_filter.cc:64] checking request: remoteAddress: 10.1.1.2:58012, localAddress: 10.1.2.66:8888, ssl: none, headers: ':authority', 'compliance-kf-system.ml'
':path', '/notebook/roger-l-c-lei/aug06/'
':method', 'GET'
'user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'
'accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
'accept-encoding', 'gzip, deflate'
'accept-language', 'en,zh-CN;q=0.9,zh;q=0.8'
'cookie', 'authservice_session=MTU5NjY5Njk0MXxOd3dBTkZvMldsVllVMUZPU0VaR01sSk5RVlJJV2xkRFVrRTFTVUl5V0RKV1EwdEhTMU5QVjFCVlUwTkpSVFpYUlVoT1RGVlBUa0U9fN3lPBXDDSZMT9MTJRbG8jv7AtblKTE3r84ayeCYuKOk; _xsrf=2|1e6639f2|10d3ea0a904e0ae505fd6425888453f8|1596697030'
'referer', 'http://compliance-kf-system.ml/jupyter/'
'upgrade-insecure-requests', '1'
'x-forwarded-for', '10.10.10.230'
'x-forwarded-proto', 'http'
'x-request-id', 'babbf884-4cec-93fd-aea6-2fc60d3abb83'
'kubeflow-userid', 'roger.l.c.lei@XXXX.com'
'x-istio-attributes', 'CjAKHWRlc3RpbmF0aW9uLnNlcnZpY2UubmFtZXNwYWNlEg8SDXJvZ2VyLWwtYy1sZWkKIwoYZGVzdGluYXRpb24uc2VydmljZS5uYW1lEgcSBWF1ZzA2Ck4KCnNvdXJjZS51aWQSQBI+a3ViZXJuZXRlczovL2lzdGlvLWluZ3Jlc3NnYXRld2F5LTg5Y2Q0YmQ0Yy1kdnF3dC5pc3Rpby1zeXN0ZW0KQQoXZGVzdGluYXRpb24uc2VydmljZS51aWQSJhIkaXN0aW86Ly9yb2dlci1sLWMtbGVpL3NlcnZpY2VzL2F1ZzA2CkMKGGRlc3RpbmF0aW9uLnNlcnZpY2UuaG9zdBInEiVhdWcwNi5yb2dlci1sLWMtbGVpLnN2Yy5jbHVzdGVyLmxvY2Fs'
'x-envoy-expected-rq-timeout-ms', '300000'
'x-b3-traceid', '3bf35cca1f7b75e7a42a046b1c124b1f'
'x-b3-spanid', 'a42a046b1c124b1f'
'x-b3-sampled', '1'
'x-envoy-original-path', '/notebook/roger-l-c-lei/aug06/'
'content-length', '0'
'x-envoy-internal', 'true'
, dynamicMetadata: filter_metadata {
  key: "istio_authn"
  value {
  }
}

[2020-08-06 13:32:43.290][26][debug][rbac] [external/envoy/source/extensions/filters/http/rbac/rbac_filter.cc:108] enforced denied

从源代码来看,允许的函数似乎返回了 false,因此它给出了“RBAC:拒绝访问”响应。

  if (engine.has_value()) {
    if (engine->allowed(*callbacks_->connection(), headers,
                        callbacks_->streamInfo().dynamicMetadata(), nullptr)) {
      ENVOY_LOG(debug, "enforced allowed");
      config_->stats().allowed_.inc();
      return Http::FilterHeadersStatus::Continue;
    } else {
      ENVOY_LOG(debug, "enforced denied");
      callbacks_->sendLocalReply(Http::Code::Forbidden, "RBAC: access denied", nullptr,
                                 absl::nullopt);
      config_->stats().denied_.inc();
      return Http::FilterHeadersStatus::StopIteration;
    }
  }

我搜索了转储的 envoy,规则似乎应该允许任何使用 header 键作为我的邮件地址的请求。现在我可以确认我已经从上面的日志 header 中得到了它。

{
 "name": "envoy.filters.http.rbac",
 "config": {
  "rules": {
   "policies": {
    "ns-access-istio": {
     "permissions": [
      {
       "and_rules": {
        "rules": [
         {
          "any": true
         }
        ]
       }
      }
     ],
     "principals": [
      {
       "and_ids": {
        "ids": [
         {
          "header": {
           "exact_match": "roger.l.c.lei@XXXX.com"
          }
         }
        ]
       }
      }
     ]
    }
   }
  }
 }
}

据了解,用于验证 RBAC authz 的 envoy 配置来自此配置。它由 mixer 分发到 sidecar,日志和代码引导我到 servicerolebinding 的 rbac.istio.io 配置。

[root@gke-client-tf leilichao]# k get servicerolebinding -n roger-l-c-lei -o yaml
apiVersion: v1
items:
- apiVersion: rbac.istio.io/v1alpha1
  kind: ServiceRoleBinding
  metadata:
    annotations:
      role: admin
      user: roger.l.c.lei@XXXX.com
    creationTimestamp: "2020-07-04T01:35:30Z"
    generation: 5
    name: owner-binding-istio
    namespace: roger-l-c-lei
    ownerReferences:
    - apiVersion: kubeflow.org/v1
      blockOwnerDeletion: true
      controller: true
      kind: Profile
      name: roger-l-c-lei
      uid: 689c9f04-08a6-4c51-a1dc-944db1a66114
    resourceVersion: "23201026"
    selfLink: /apis/rbac.istio.io/v1alpha1/namespaces/roger-l-c-lei/servicerolebindings/owner-binding-istio
    uid: bbbffc28-689c-4099-837a-87a2feb5948f
  spec:
    roleRef:
      kind: ServiceRole
      name: ns-access-istio
    subjects:
    - properties:
        request.headers[]: roger.l.c.lei@XXXX.com
  status: {}
kind: List
metadata:
  resourceVersion: ""
  selfLink: ""

我想尝试更新此 ServiceRoleBinding 以验证一些假设,因为我无法调试 envoy 源代码并且没有足够的日志来显示“允许”方法返回 false 的确切原因。

但是我发现自己无法更新服务角色绑定。每次编辑完都会恢复到原来的版本

我发现有这个 istio-galley validatingAdmissionConfiguration(下面的代码块)监控这些 istio rbac 资源。

[root@gke-client-tf leilichao]# k get validatingwebhookconfigurations istio-galley -oyaml
apiVersion: admissionregistration.k8s.io/v1beta1
kind: ValidatingWebhookConfiguration
metadata:
  creationTimestamp: "2020-08-04T15:00:59Z"
  generation: 1
  labels:
    app: galley
    chart: galley
    heritage: Tiller
    istio: galley
    release: istio
  name: istio-galley
  ownerReferences:
  - apiVersion: extensions/v1beta1
    blockOwnerDeletion: true
    controller: true
    kind: Deployment
    name: istio-galley
    uid: 11fef012-4145-49ac-a43c-2e1d0a460ea4
  resourceVersion: "22484680"
  selfLink: /apis/admissionregistration.k8s.io/v1beta1/validatingwebhookconfigurations/istio-galley
  uid: 6f485e28-3b5a-4a3b-b31f-a5c477c82619
webhooks:
- admissionReviewVersions:
  - v1beta1
  clientConfig:
    caBundle: 
    .
    .
    .
    service:
      name: istio-galley
      namespace: istio-system
      path: /admitpilot
      port: 443
  failurePolicy: Fail
  matchPolicy: Exact
  name: pilot.validation.istio.io
  namespaceSelector: {}
  objectSelector: {}
  rules:
  - apiGroups:
    - config.istio.io
    apiVersions:
    - v1alpha2
    operations:
    - CREATE
    - UPDATE
    resources:
    - httpapispecs
    - httpapispecbindings
    - quotaspecs
    - quotaspecbindings
    scope: '*'
  - apiGroups:
    - rbac.istio.io
    apiVersions:
    - '*'
    operations:
    - CREATE
    - UPDATE
    resources:
    - '*'
    scope: '*'
  - apiGroups:
    - authentication.istio.io
    apiVersions:
    - '*'
    operations:
    - CREATE
    - UPDATE
    resources:
    - '*'
    scope: '*'
  - apiGroups:
    - networking.istio.io
    apiVersions:
    - '*'
    operations:
    - CREATE
    - UPDATE
    resources:
    - destinationrules
    - envoyfilters
    - gateways
    - serviceentries
    - sidecars
    - virtualservices
    scope: '*'
  sideEffects: Unknown
  timeoutSeconds: 30
- admissionReviewVersions:
  - v1beta1
  clientConfig:
    caBundle: 
    .
    .
    .
    service:
      name: istio-galley
      namespace: istio-system
      path: /admitmixer
      port: 443
  failurePolicy: Fail
  matchPolicy: Exact
  name: mixer.validation.istio.io
  namespaceSelector: {}
  objectSelector: {}
  rules:
  - apiGroups:
    - config.istio.io
    apiVersions:
    - v1alpha2
    operations:
    - CREATE
    - UPDATE
    resources:
    - rules
    - attributemanifests
    - circonuses
    - deniers
    - fluentds
    - kubernetesenvs
    - listcheckers
    - memquotas
    - noops
    - opas
    - prometheuses
    - rbacs
    - solarwindses
    - stackdrivers
    - cloudwatches
    - dogstatsds
    - statsds
    - stdios
    - apikeys
    - authorizations
    - checknothings
    - listentries
    - logentries
    - metrics
    - quotas
    - reportnothings
    - tracespans
    scope: '*'
  sideEffects: Unknown
  timeoutSeconds: 30

长与短

我已经为这个 istio 问题苦苦思索了 2 个多星期。我敢肯定,有很多人在尝试在 k8s 上对 istio 进行故障排除时有同样的感觉。欢迎任何建议! 以上是我对问题的理解,如有错误请指正:

我 运行 遇到以下问题

即使删除了验证 webhook,我也无法更新 ServiceRoleBinding

我试图删除这个验证 webhook 来更新 servicerolebinding。资源在我保存编辑后立即恢复。 验证 webhook 实际上是从 configmap 自动生成的,所以我必须更新它来更新 webhook。

Mixer 中是否有某种缓存用于分发配置

我找不到任何相关日志表明 rbac.istio.io 资源被 istio-system 命名空间中的任何服务 protected/validated。

如何获取 MIXER 的日志

我需要了解哪个组件确切地控制了策略。我设法更新了日志级别,但没有找到任何有用的东西

最重要的是如何调试 envoy 容器

我需要调试 envoy 应用程序以了解为什么它会为 allow 函数返回 false。 如果我们不能轻易调试它。有没有文档可以让我更新代码以添加更多日志并为 GCR 构建新图像,这样我就可以拥有另一个 运行 并根据日志查看幕后发生的事情。

回答我自己的问题,因为我在这些问题上取得了一些进展。

即使删除了验证 webhook,我也无法更新 ServiceRoleBinding

那是因为 ServiceRoleBinding 实际上是由 kubeflow 命名空间中的 配置文件控制器 generated/monitored/managed 而不是 validating webhook

我遇到了这个 rbac 问题,因为基于配置文件清单文件夹中的 params.yaml,规则生成为

request.headers[]: roger.l.c.lei@XXXX.com

而不是

request.headers[kubeflow-userid]: roger.l.c.lei@XXXX.com

由于我 mis-configed 的值是空白而不是 userid-header=kubeflow-userid 在 params.yaml

检查您的应用程序命名空间中的 authorizationpolicy 资源。

对于新集群,请参阅问题 4440 中的评论

https://github.com/kubeflow/pipelines/issues/4440

cat << EOF | kubectl apply -f -
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
 name: bind-ml-pipeline-nb-kubeflow-user-example-com
 namespace: kubeflow
spec:
 selector:
   matchLabels:
     app: ml-pipeline
 rules:
 - from:
   - source:
       principals: ["cluster.local/ns/kubeflow-user-example-com/sa/default-editor"]
---
apiVersion: networking.istio.io/v1alpha3
kind: EnvoyFilter
metadata:
  name: add-header
  namespace: kubeflow-user-example-com
spec:
  configPatches:
  - applyTo: VIRTUAL_HOST
    match:
      context: SIDECAR_OUTBOUND
      routeConfiguration:
        vhost:
          name: ml-pipeline.kubeflow.svc.cluster.local:8888
          route:
            name: default
    patch:
      operation: MERGE
      value:
        request_headers_to_add:
        - append: true
          header:
            key: kubeflow-userid
            value: user@example.com
  workloadSelector:
    labels:
      notebook-name: test2
EOF

在我的笔记本里

import kfp
client = kfp.Client()
print(client.list_experiments())

输出

{'experiments': [{'created_at': datetime.datetime(2021, 8, 12, 9, 14, 20, tzinfo=tzlocal()),
                  'description': None,
                  'id': 'b2e552e5-3324-483a-8ec8-b32894f49281',
                  'name': 'test',
                  'resource_references': [{'key': {'id': 'kubeflow-user-example-com',
                                                   'type': 'NAMESPACE'},
                                           'name': None,
                                           'relationship': 'OWNER'}],
                  'storage_state': 'STORAGESTATE_AVAILABLE'}],
 'next_page_token': None,
 'total_size': 1}