Karmada 有趣的玩法：ANP

avatar 我们都知道 Karmada 纳管成员集群有 push 和 pull 两种模式，pull 模式主要用于解决控制面与数据面的网络单向通信的应用场景。比如 Karmada 控制面部署在云上，成员集群在私有环境没有固定的公网IP或出于安全考虑，成员集群的 ApiServer 不对外开放。

为了更方便管理 pull 模式的成员集群，Karmada 也提供 ANP (apiserver-network-proxy) 解决方案，通过 clusters/proxy 来转发请求。 ANP 是在 kubernetes 的 EgressSelector 功能上开发的一个参考实现，旨在实现 Kubernetes 集群组件的跨网络通信。比如 Konnectivity 可以为控制面提供集群通信的 TCP 级别代理。

接下来笔者举个简单的例子，在 Karmada 控制面通过 ANP，获取 pull 模式成员集群的 pod 日志。

部署 ANP

克隆 ANP 仓库

这里用的是 v0.0.30 版本

git clone --branch v0.0.30 https://github.com/kubernetes-sigs/apiserver-network-proxy.git

生成证书

PROXY_SERVER_HOST 是 Karmadahost 集群的 IP

cd apiserver-network-proxy
make certs PROXY_SERVER_IP={PROXY_SERVER_HOST}

在 Karmada host 集群部署 proxy server

复制证书到 /etc/anp-server 目录

mkdir  /etc/anp-server
cp certs/frontend/issued/ca.crt  /etc/anp-server/server-ca.crt
cp certs/frontend/issued/proxy-frontend.crt /etc/anp-server/server-proxy-frontend.crt
cp certs/frontend/private/proxy-frontend.key /etc/anp-server/server-proxy-frontend.key
cp certs/agent/issued/ca.crt /etc/anp-server/cluster-ca.crt
cp certs/agent/issued/proxy-frontend.crt /etc/anp-server/cluster-proxy-frontend.crt
cp certs/agent/private/proxy-frontend.key /etc/anp-server/cluster-proxy-frontend.key

{PROXY_SERVER_NAME} 是证书所在的 Node Name 默认模式是 GRPC，可以通过 –mode=http-connect 切换为 http-connect 模式

apiVersion: apps/v1
kind: Deployment
metadata:
  name: proxy-server
  namespace: karmada-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: proxy-server
  template:
    metadata:
      labels:
        app: proxy-server
    spec:
      containers:
      - args:
          - --health-port=8092
          - --proxy-strategies=destHost
          - --server-ca-cert=/certs/server-ca.crt
          - --server-cert=/certs/server-proxy-frontend.crt
          - --server-key=/certs/server-proxy-frontend.key
          - --cluster-ca-cert=/certs/cluster-ca.crt
          - --cluster-cert=/certs/cluster-proxy-frontend.crt
          - --cluster-key=/certs/cluster-proxy-frontend.key
        image: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server:v0.0.30
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 8092
            scheme: HTTP
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 60
        name: proxy-server
        volumeMounts:
        - mountPath: /certs
          name: cert
      restartPolicy: Always
      hostNetwork: true
      nodeSelector:
        kubernetes.io/hostname: {PROXY_SERVER_NAME}
      volumes:
      - name: cert
        hostPath:
          path: /etc/anp-server

部署 proxy server

kubectl apply -f proxy-server.yaml

在成员集群部署 proxy agent

打包 agent 证书并上传到成员集群

mkdir anp-agent
cp certs/agent/issued/ca.crt anp-agent/ca.crt
cp certs/agent/issued/proxy-agent.crt anp-agent/proxy-agent.crt
cp certs/agent/private/proxy-agent.key anp-agent/proxy-agent.key
tar -zcvf anp-agent.tar.gz anp-agent

在成员集群解压证书

tar -zxvf anp-agent.tar.gz -C /etc

{PROXY_AGENT_NAME} 是证书所在的 Node Name {PROXY_SERVER_HOST} 是 proxy server IP

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: proxy-agent
  name: proxy-agent
  namespace: karmada-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: proxy-agent
  template:
    metadata:
      labels:
        app: proxy-agent
    spec:
      containers:
      - args:
        - --ca-cert=/certs/ca.crt
        - --agent-cert=/certs/proxy-agent.crt
        - --agent-key=/certs/proxy-agent.key
        - --proxy-server-host={PROXY_SERVER_HOST}
        - --proxy-server-port=8091
        - --agent-identifiers=host=${HOST_IP}
        image: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent:v0.0.30
        imagePullPolicy: IfNotPresent
        name: proxy-agent
        env:
          - name: HOST_IP
            valueFrom:
              fieldRef:
                fieldPath: status.hostIP
        livenessProbe:
          httpGet:
            scheme: HTTP
            port: 8093
            path: /healthz
          initialDelaySeconds: 15
          timeoutSeconds: 60
        volumeMounts:
          - mountPath: /certs
            name: cert
      nodeSelector:
        kubernetes.io/hostname: {PROXY_AGENT_NAME}
      volumes:
        - name: cert
          hostPath:
            path: /etc/anp-agent

部署 proxy agent

kubectl apply -f proxy-agent.yaml

获取 pull 模式成员集群 Pod 日志

先通过 konnectivity 库创建一个 Grpc Tunnel

func (o *LogsPullOptions) CreateTunnel() (konnectivity.Tunnel, error) {
 tlsCfg, err := util.GetClientTLSConfig(o.ProxyCACert, o.ProxyCert, o.ProxyKey, o.ProxyServerHost, nil)
 if err != nil {
  return nil, err
 }

 return konnectivity.CreateSingleUseGrpcTunnel(
  context.TODO(),
  net.JoinHostPort(o.ProxyServerHost, o.ProxyServerPort),
  grpc.WithTransportCredentials(grpccredentials.NewTLS(tlsCfg)),
  grpc.WithKeepaliveParams(keepalive.ClientParameters{
   Time: time.Second * 5,
  }),
 )
}

设置 kubeconfig，请求走 Grpc Tunnel

 cfg, err := util.RestConfig(false, o.GlobalOptions.Kubeconfig)
 if err != nil {
  return err
 }
 cfg.Dial = dialerTunnel.DialContext

获取日志

func (o *LogsPullOptions) GetPodLogs(client *kubernetes.Clientset) error {
 podLogOpts := corev1.PodLogOptions{
  Follow: o.Follow,
 }
 if o.TailLines != 0 {
  podLogOpts.TailLines = &o.TailLines
 }

 reqLogs := client.CoreV1().Pods(o.GlobalOptions.Namespace).GetLogs(o.PodName, &podLogOpts)
 podLogs, err := reqLogs.Stream(context.TODO())
 if err != nil {
  return err
 }
 defer podLogs.Close()

 r := bufio.NewReader(podLogs)
 for {
  b, err := r.ReadBytes('\n')
  if err != nil {
   return err
  }
  fmt.Print(string(b))
 }
}