背景

在多分支并行开发的集群下，新建分支以及删除分支都需要开发人员手动维护Istio/K8s的资源对象

目的

结合git hook -》 sync程序 -》 kubernetes API 流程，自动维护k8s以及istio资源对象，减少成本，提高开发效率

技术栈

GitLab Api

使用GitLab Api，获取项目分支信息

1
2
3
4
5
6
 <!-- https://mvnrepository.com/artifact/org.gitlab/java-gitlab-api -->
<dependency>
    <groupId>org.gitlab</groupId>
    <artifactId>java-gitlab-api</artifactId>
    <version>${gitlab.version}</version>
</dependency>

gitlab api认证

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
/**
 * yaml: 
 * gitlab-config:
 *   url: http://xxxxx.com
 *   token: xxxxxxxxxxxxxxxx
 */
@Data
@Configuration
@ConfigurationProperties(prefix = "gitlab-config")
public class GitLabConfiguration {
    private String url;
    private String token;
    private String hookToken;
    @Bean
    public GitlabAPI gitlabAPI() {
        return GitlabAPI.connect(url, token);
    }
}

gitlab hook配置

1
2
3
请求体如下：主要会用到的参数有before，after，event_name，project_id，ref(分支名)，project.name

{"object_kind":"push","event_name":"push","before":"f13ce2dcdab3318fc528d281e2a76533dc289026","after":"eeb9a8ccd5b34111dee7f5969e0d15d3092f754d","ref":"refs/heads/feature-amy","checkout_sha":"eeb9a8ccd5b34111dee7f5969e0d15d3092f754d","user_id":102,"user_name":"amy","user_username":"amy","user_email":"baimeiling@sopeiyun.com","user_avatar":"http://127.0.0.1:18090/uploads/-/system/user/avatar/102/avatar.png","project_id":156,"project":{"id":156,"name":"app_tenant","description":"","web_url":"http://127.0.0.1:18090/back-end/app_tenant","git_ssh_url":"git@127.0.0.1:back-end/app_tenant.git","git_http_url":"http://127.0.0.1:18090/back-end/app_tenant.git","namespace":"back-end","visibility_level":0,"path_with_namespace":"back-end/app_tenant","default_branch":"master","homepage":"http://127.0.0.1:18090/back-end/app_tenant","url":"git@127.0.0.1:back-end/app_tenant.git","ssh_url":"git@127.0.0.1:back-end/app_tenant.git","http_url":"http://127.0.0.1:18090/back-end/app_tenant.git"},"commits":[{"id":"eeb9a8ccd5b34111dee7f5969e0d15d3092f754d","message":"订单管理：修改商品查询来源格式\n","timestamp":1676884364000,"url":"http://127.0.0.1:18090/back-end/app_tenant/commit/eeb9a8ccd5b34111dee7f5969e0d15d3092f754d","author":{"name":"amy","email":"baimeiling@sopeiyun.com"},"added":[],"modified":["controllers/product.js"],"removed":[]}],"total_commits_count":1,"repository":{"name":"app_tenant","url":"git@127.0.0.1:back-end/app_tenant.git","description":"","homepage":"http://127.0.0.1:18090/back-end/app_tenant","git_http_url":"http://127.0.0.1:18090/back-end/app_tenant.git","git_ssh_url":"git@127.0.0.1:back-end/app_tenant.git","visibility_level":0}}

fabric8

This client provides access to the full Kubernetes & OpenShift REST APIs via a fluent DSL.

1
2
3
4
5
6
7
8
9
10
11
12
<!-- 对应版本：https://github.com/fabric8io/kubernetes-client/#kubernetes-compatibility-matrix -->
<dependency>
    <groupId>io.fabric8</groupId>
    <artifactId>kubernetes-client</artifactId>
    <version>${kubernetes-client.version}</version>
</dependency>
<!-- 因为要操作istio，所以需要istio拓展 -->
<dependency>
    <groupId>io.fabric8</groupId>
    <artifactId>istio-client</artifactId>
    <version>${kubernetes-client.version}</version>
</dependency>

Client认证

当与集群交互的时候常用的Client认证方式包括kubeconfig（即证书） 和 token，以下使用证书方式，默认在~/.kube/config，参考站内使用kubeconfig或token进行用户身份认证

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
@Data
@Configuration
public class SyncGitToIstioConfiguration {
    @Bean(destroyMethod = "close")
    public IstioClient istioClient() throws IOException {
        // 读取Kube Config连接配置文件。
        Resource resourceAsStream = new ClassPathResource("kube/config");
        String kubeconfigContents = IOHelpers.readFully(resourceAsStream.getInputStream());
        Config config = Config.fromKubeconfig(null, kubeconfigContents, null);
        //Config config = new ConfigBuilder()
        //        .withOauthToken(token)
        //        .withTrustCerts(true)
        //        .withMasterUrl("https://" + k8sMasterIp + ":6443/")
        //        .build();
        return new DefaultIstioClient(config);
    }
    @Bean(destroyMethod = "close")
    public KubernetesClient kubernetesClient() throws IOException {
        // 读取Kube Config连接配置文件。
        Resource resourceAsStream = new ClassPathResource("kube/config");
        String kubeconfigContents = IOHelpers.readFully(resourceAsStream.getInputStream());
        Config config = Config.fromKubeconfig(null, kubeconfigContents, null);
        //Config config = new ConfigBuilder()
        //        .withOauthToken(token)
        //        .withTrustCerts(true)
        //        .withMasterUrl("https://" + k8sMasterIp + ":6443/")
        //        .build();
        return new KubernetesClientBuilder()
                .withConfig(config)
                .build();
    }
}

需求

如果token认证失败，则return
如果before是0，则为create branch：1.取所有分支名 2.default分支为旧分支 3.createOrUpdate istio 对象
如果after是0，则为delete branch：1.取所有分支名 2.default分支为旧分支 3.createOrUpdate istio 对象 4.删除k8s deploy对象
如果是新项目（空的virtual service）：1.取所有分支名 2.default分支为master分支 3.createOrUpdate istio 对象
如果before和after都不是0，说明是普通的push操作，则return
如果push操作的分支名称不以release开头，则return
如果当前是delete branch操作，且branch.equals(default分支)，则将default分支修改为master

代码实现

yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: test
  namespace: test-istio
  labels:
    app: test
spec:
  host: test.test-istio.svc.cluster.local
  subsets:
    - labels:
        version: master
      name: master
    - labels:
        version: test
      name: test
---
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: test
  namespace: test-istio
  labels:
    app: test
spec:
  hosts:
    - test.test-istio.svc.cluster.local
  http:
    - match:
        - headers:
            project-version:
              exact: master
      route:
        - destination:
            host: test.test-istio.svc.cluster.local
            subset: master
    - match:
        - headers:
            project-version:
              exact: test
      route:
        - destination:
            host: test.test-istio.svc.cluster.local
            subset: test
    - route:
        - destination:
            host: test.test-istio.svc.cluster.local
            subset: master

yaml -> object 代码实现

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
// 保证并发请求下, 同一个项目的幂等操作
synchronized (String.valueOf(projectId).intern()) {
            if (isCreateBranch) {
                createOrUpdateVsAndDr(projectName, namespace, host, branches, true, currentBranch);
            }
            // if delete operation, createOrUpdate istio object, then delete deployment object
            else {
                createOrUpdateVsAndDr(projectName, namespace, host, branches, false, currentBranch);
                kubernetesClient.apps().deployments()
                        .inNamespace(namespace)
                        .withName(projectName + "-" + currentBranch)
                        .delete();
            }
        }
// createOrUpdate vs/dr对象
private void createOrUpdateVsAndDr(String projectName, String namespace, String host, List<String> branches, boolean isCreateBranch, String currentBranch) {
    List<Subset> subsets = new ArrayList<>();
    branches.forEach(subset -> {
        // add current subset
        Subset sub = new Subset();
        sub.setName(subset);
        Map<String, String> subMap = new HashMap<>(1);
        subMap.put("version", subset);
        sub.setLabels(subMap);
        subsets.add(sub);
    });
    // add master subset
    Subset sub = new Subset();
    sub.setName("master");
    Map<String, String> subMap = new HashMap<>(1);
    subMap.put("version", "master");
    sub.setLabels(subMap);
    subsets.add(sub);
    // build DestinationRule
    DestinationRule destinationRule = new DestinationRuleBuilder()
        .withNewMetadata()
        .withName(projectName)
        .withNamespace(namespace)
        .addToLabels("app", projectName)
        .endMetadata()
        .withNewSpec()
        .withHost(String.format(host, projectName))
        .withSubsets(subsets)
        .endSpec()
        .build();
    // createOrReplace DestinationRule
    istioClient.v1beta1().destinationRules().inNamespace(namespace).resource(destinationRule).createOrReplace();

    // The total sum of the route weights should be 100(or do not configure the weight), otherwise the admission hook has a validation error when creating or patching the resource
    // https://github.com/istio/istio/issues/36021, https://kubebyexample.com/learning-paths/istio/traffic-management, https://istio.io/latest/zh/docs/ops/common-problems/validation/
    // build VirtualService
    VirtualServiceFluent.SpecNested<VirtualServiceBuilder> virtualApp = new VirtualServiceBuilder()
        .withNewMetadata()
        .withName(projectName)
        .withNamespace(namespace)
        .addToLabels("app", projectName)
        .endMetadata()
        .withNewSpec()
        .addToHosts(String.format(host, projectName));


    branches.forEach(subset -> {
        Map<String, StringMatch> headers = new HashMap<>(1);
        headers.put("project-version", new StringMatch(new StringMatchExact(subset)));
        Destination destination = new Destination();
        destination.setHost(String.format(host, projectName));
        destination.setSubset(subset);
        virtualApp
            .addNewHttp()
            .addNewMatch().withHeaders(headers).endMatch()
            .addNewRoute().withDestination(destination)
            .endRoute()
            .endHttp();
    });

    // default Virtual Service Route
    VirtualService defaultVirtualService = istioClient.v1beta1()
        .virtualServices()
        .inNamespace(namespace)
        .withName(projectName)
        .get();
    //log.info("打印defaultVirtualService：{}", ContentConvertUtils.jsonToYaml(JSONUtil.toJsonStr(defaultVirtualService)));
    if (Objects.isNull(defaultVirtualService)) {
        // null Virtual Service resource -> is new project -> add master branch
        virtualApp
            .addNewHttp()
            .addNewRoute()
            .withNewDestination()
            .withHost(String.format(host, projectName))
            // default branch
            .withSubset("master")
            .endDestination()
            .endRoute()
            .endHttp();
    } else {
        // has already created a resource, so add old resource route
        List<HTTPRoute> http = defaultVirtualService.getSpec().getHttp();
        List<HTTPRouteDestination> route = http.get(http.size() - 1).getRoute();
        HTTPRouteDestination defaultHttpRouteDestination = route.get(route.size() - 1);
        // 如果当前是delete branch操作，且branch.equals(default分支)，则将default分支修改为master
        String defaultBranch = defaultHttpRouteDestination.getDestination().getSubset();
        String defaultHost = defaultHttpRouteDestination.getDestination().getHost();
        if (!isCreateBranch && currentBranch.equals(defaultBranch)){
            defaultHost = String.format(host, projectName);
            defaultBranch = "master";
        }
        virtualApp
            .addNewHttp()
            .addNewRoute()
            .withNewDestination()
            .withHost(defaultHost)
            // default branch
            .withSubset(defaultBranch)
            .endDestination()
            .endRoute()
            .endHttp();
    }
    VirtualService virtualService = virtualApp
        .endSpec()
        .build();

    //log.info("打印virtualService：{}", ContentConvertUtils.jsonToYaml(JSONUtil.toJsonStr(virtualService)));
    // createOrReplace VirtualService
    istioClient.v1beta1().virtualServices().inNamespace(namespace).resource(virtualService).createOrReplace();
}

参考

https://kubebyexample.com/learning-paths/istio/traffic-management

https://github.com/fabric8io/kubernetes-client

https://github.com/fabric8io/kubernetes-client/tree/master/extensions/istio/examples