Skip to content

Commit 167d452

Browse files
sudharshanibmsudharshanibm
committed
Add terraform for provisioning s390x build cluster on ibmcloud
Signed-off-by: Sudharshan Muralidharan <[email protected]>
1 parent 5c99c23 commit 167d452

File tree

26 files changed

+1388
-0
lines changed

26 files changed

+1388
-0
lines changed

‎infra/ibmcloud/terraform/k8s-s390x-build-cluster/bastion.tf‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
resource "ibm_is_instance" "bastion" {
17+
count = var.bastion.count
18+
name = "bastion-s390x-${count.index + 1}"
19+
vpc = data.ibm_is_vpc.vpc.id
20+
zone = var.zone
21+
profile = var.bastion.profile
22+
image = var.image_id
23+
keys = [ibm_is_ssh_key.k8s_ssh_key.id]
24+
resource_group = data.ibm_resource_group.resource_group.id
25+
primary_network_interface {
26+
name = "public-nic"
27+
subnet = data.ibm_is_subnet.subnet.id
28+
security_groups = [data.ibm_is_security_group.bastion_sg.id]
29+
}
30+
31+
boot_volume {
32+
name = "boot-vol-bastion-${count.index}"
33+
size = var.bastion.boot_volume.size
34+
}
35+
36+
user_data = <<-EOF
37+
#cloud-config
38+
package_update: true
39+
package_upgrade: true
40+
packages:
41+
- tcpdump
42+
- net-tools
43+
- iptables-persistent
44+
write_files:
45+
- path: /etc/ssh/sshd_config.d/99-bastion.conf
46+
content: |
47+
AllowTcpForwarding yes
48+
GatewayPorts yes
49+
PermitTunnel yes
50+
PermitRootLogin prohibit-password
51+
PasswordAuthentication no
52+
ClientAliveInterval 120
53+
ClientAliveCountMax 3
54+
MaxSessions 50
55+
MaxStartups 50:30:100
56+
- path: /etc/systemd/network/10-eth1.network
57+
content: |
58+
[Match]
59+
Name=eth1
60+
[Network]
61+
Address=${data.ibm_is_subnet.subnet.ipv4_cidr_block}
62+
DNS=8.8.8.8
63+
DNS=8.8.4.4
64+
runcmd:
65+
- [sysctl, -w, net.ipv4.ip_forward=1]
66+
- [echo, "net.ipv4.ip_forward = 1", ">>", /etc/sysctl.conf]
67+
- [iptables, -t, nat, -A, POSTROUTING, -o, eth0, -j, MASQUERADE]
68+
- [iptables, -A, FORWARD, -i, eth1, -o, eth0, -j, ACCEPT]
69+
- [iptables, -A, FORWARD, -i, eth0, -o, eth1, -m, state, --state, RELATED,ESTABLISHED, -j, ACCEPT]
70+
- [netfilter-persistent, save]
71+
- [systemctl, restart, systemd-networkd]
72+
- [systemctl, restart, sshd]
73+
- [hostnamectl, set-hostname, "bastion-s390x-${count.index + 1}.s390x-vpc.cloud.ibm.com"]
74+
- [echo, "bastion-s390x-${count.index + 1}.s390x-vpc.cloud.ibm.com", ">", /etc/hostname]
75+
- [sed, -i, "s/^127.0.1.1.*/127.0.1.1\tbastion-s390x-${count.index + 1}.s390x-vpc.cloud.ibm.com/", /etc/hosts]
76+
EOF
77+
}
78+
79+
resource "ibm_is_floating_ip" "bastion_fip" {
80+
count = var.bastion.count
81+
name = "bastion-fip-${count.index}"
82+
target = ibm_is_instance.bastion[count.index].primary_network_interface[0].id
83+
resource_group = data.ibm_resource_group.resource_group.id
84+
}
85+
86+
resource "time_sleep" "wait_for_bastion" {
87+
count = var.bastion.count
88+
depends_on = [ibm_is_floating_ip.bastion_fip]
89+
90+
create_duration = "180s" # Wait 3 minutes for full initialization
91+
}
92+
93+
resource "null_resource" "bastion_setup" {
94+
count = var.bastion.count
95+
depends_on = [time_sleep.wait_for_bastion]
96+
97+
connection {
98+
type = "ssh"
99+
user = "root"
100+
host = ibm_is_floating_ip.bastion_fip[count.index].address
101+
private_key = data.ibm_sm_arbitrary_secret.ssh_private_key.payload
102+
timeout = "5m"
103+
}
104+
105+
provisioner "remote-exec" {
106+
inline = [
107+
# Network verification
108+
"echo '=== Network Interfaces ==='",
109+
"ip -4 addr show",
110+
"echo '=== Routing Table ==='",
111+
"ip route",
112+
"echo '=== NAT Configuration ==='",
113+
"iptables -t nat -L -n -v",
114+
"echo '=== IP Forwarding ==='",
115+
"sysctl net.ipv4.ip_forward",
116+
117+
# Hostname verification
118+
"echo '=== Hostname ==='",
119+
"hostname",
120+
"hostnamectl",
121+
"cat /etc/hostname",
122+
123+
# Final security updates
124+
"command -v apt-get >/dev/null && apt-get update -y && apt-get upgrade -y --security || true",
125+
"command -v yum >/dev/null && yum update -y --security || true",
126+
"command -v dnf >/dev/null && dnf update -y --security || true"
127+
]
128+
}
129+
}

‎infra/ibmcloud/terraform/k8s-s390x-build-cluster/data.tf‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
data "ibm_resource_group" "resource_group" {
17+
name = "rg-build-cluster"
18+
}
19+
20+
data "ibm_is_vpc" "vpc" {
21+
name = "k8s-s390x-vpc"
22+
}
23+
24+
data "ibm_is_subnet" "subnet" {
25+
name = "k8s-s390x-subnet"
26+
}
27+
28+
data "ibm_is_security_group" "bastion_sg" {
29+
name = "k8s-vpc-s390x-bastion-sg"
30+
vpc = data.ibm_is_vpc.vpc.id
31+
}
32+
33+
data "ibm_is_security_group" "master_sg" {
34+
name = "k8s-vpc-s390x-master-sg"
35+
vpc = data.ibm_is_vpc.vpc.id
36+
}
37+
38+
data "ibm_is_security_group" "worker_sg" {
39+
name = "k8s-vpc-s390x-worker-sg"
40+
vpc = data.ibm_is_vpc.vpc.id
41+
}
42+
43+
data "ibm_sm_arbitrary_secret" "ssh_private_key" {
44+
instance_id = var.secrets_manager_id
45+
region = var.region
46+
name = "zvsi-ssh-private-key"
47+
secret_group_name = "default"
48+
}
49+
50+
data "ibm_sm_arbitrary_secret" "ssh_public_key" {
51+
instance_id = var.secrets_manager_id
52+
region = var.region
53+
name = "zvsi-ssh-public-key"
54+
secret_group_name = "default"
55+
}

‎infra/ibmcloud/terraform/k8s-s390x-build-cluster/load_balancer.tf‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
resource "ibm_is_lb" "k8s_load_balancer" {
17+
name = "k8s-s390x-lb"
18+
type = "public"
19+
subnets = [data.ibm_is_subnet.subnet.id]
20+
resource_group = data.ibm_resource_group.resource_group.id
21+
security_groups = [data.ibm_is_security_group.master_sg.id]
22+
}
23+
24+
resource "ibm_is_lb_pool" "k8s_api_pool" {
25+
name = "k8s-api-server-pool"
26+
lb = ibm_is_lb.k8s_load_balancer.id
27+
protocol = "tcp"
28+
algorithm = "round_robin"
29+
health_delay = 5
30+
health_retries = 2
31+
health_timeout = 2
32+
health_type = "tcp"
33+
health_monitor_url = "/"
34+
health_monitor_port = 6443
35+
}
36+
37+
resource "ibm_is_lb_listener" "k8s_api_listener" {
38+
lb = ibm_is_lb.k8s_load_balancer.id
39+
protocol = "tcp"
40+
port = 6443
41+
default_pool = ibm_is_lb_pool.k8s_api_pool.pool_id
42+
}
43+
44+
resource "ibm_is_lb_pool_member" "k8s_api_members" {
45+
count = var.control_plane.count
46+
lb = ibm_is_lb.k8s_load_balancer.id
47+
pool = ibm_is_lb_pool.k8s_api_pool.pool_id
48+
port = 6443
49+
target_address = ibm_is_instance.control_plane[count.index].primary_network_interface[0].primary_ipv4_address
50+
}

‎infra/ibmcloud/terraform/k8s-s390x-build-cluster/nodes.tf‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
resource "ibm_is_ssh_key" "k8s_ssh_key" {
17+
name = "k8s-s390x-ssh-key"
18+
public_key = data.ibm_sm_arbitrary_secret.ssh_public_key.payload
19+
resource_group = data.ibm_resource_group.resource_group.id
20+
}
21+
22+
resource "ibm_is_instance" "control_plane" {
23+
count = var.control_plane.count
24+
name = "master-s390x-${count.index + 1}"
25+
vpc = data.ibm_is_vpc.vpc.id
26+
zone = var.zone
27+
profile = var.control_plane.profile
28+
image = var.image_id
29+
keys = [ibm_is_ssh_key.k8s_ssh_key.id]
30+
resource_group = data.ibm_resource_group.resource_group.id
31+
32+
primary_network_interface {
33+
subnet = data.ibm_is_subnet.subnet.id
34+
security_groups = [data.ibm_is_security_group.master_sg.id]
35+
}
36+
37+
boot_volume {
38+
name = "boot-vol-master-${count.index + 1}"
39+
size = var.control_plane.boot_volume.size
40+
}
41+
}
42+
43+
resource "ibm_is_instance" "compute" {
44+
count = var.compute.count
45+
name = "worker-s390x-${count.index + 1}"
46+
vpc = data.ibm_is_vpc.vpc.id
47+
zone = var.zone
48+
profile = var.compute.profile
49+
image = var.image_id
50+
keys = [ibm_is_ssh_key.k8s_ssh_key.id]
51+
resource_group = data.ibm_resource_group.resource_group.id
52+
53+
primary_network_interface {
54+
subnet = data.ibm_is_subnet.subnet.id
55+
security_groups = [data.ibm_is_security_group.worker_sg.id]
56+
}
57+
58+
boot_volume {
59+
name = "boot-vol-worker-${count.index + 1}"
60+
size = var.compute.boot_volume.size
61+
}
62+
}
63+
64+
resource "null_resource" "node_setup" {
65+
for_each = merge(
66+
{ for idx, instance in ibm_is_instance.control_plane : "master-${idx}" => instance },
67+
{ for idx, instance in ibm_is_instance.compute : "worker-${idx}" => instance }
68+
)
69+
depends_on = [time_sleep.wait_for_bastion]
70+
triggers = {
71+
instance_id = each.value.id
72+
retry_count = 0
73+
}
74+
connection {
75+
type = "ssh"
76+
user = "root"
77+
host = each.value.primary_network_interface[0].primary_ipv4_address
78+
private_key = data.ibm_sm_arbitrary_secret.ssh_private_key.payload
79+
bastion_host = ibm_is_floating_ip.bastion_fip[0].address
80+
timeout = "5m"
81+
agent = false
82+
83+
}
84+
}

‎infra/ibmcloud/terraform/k8s-s390x-build-cluster/outputs.tf‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
output "bastion_public_ips" {
17+
description = "Public Floating IP(s) assigned to the bastion"
18+
value = ibm_is_floating_ip.bastion_fip[*].address
19+
}
20+
21+
output "bastion_private_ips" {
22+
description = "Private IP address(es) of the bastion host(s)"
23+
value = ibm_is_instance.bastion[*].primary_network_interface[0].primary_ipv4_address
24+
}
25+
26+
output "master_node_ips" {
27+
description = "Private IP addresses of control plane nodes"
28+
value = ibm_is_instance.control_plane[*].primary_network_interface[0].primary_ipv4_address
29+
}
30+
31+
output "worker_node_ips" {
32+
description = "Private IP addresses of worker nodes"
33+
value = ibm_is_instance.compute[*].primary_network_interface[0].primary_ipv4_address
34+
}
35+
36+
output "api_load_balancer_hostname" {
37+
description = "Hostname of the Kubernetes API load balancer"
38+
value = ibm_is_lb.k8s_load_balancer.hostname
39+
}
40+
output "subnet_cidr" {
41+
description = "CIDR block of the public subnet"
42+
value = data.ibm_is_subnet.subnet.ipv4_cidr_block
43+
}

‎infra/ibmcloud/terraform/k8s-s390x-build-cluster/providers.tf‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
Copyright 2025 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
locals {
17+
key = var.ibmcloud_api_key
18+
region = "eu-de"
19+
zone = "eu-de-1"
20+
}
21+
22+
provider "ibm" {
23+
ibmcloud_api_key = local.key
24+
region = local.region
25+
zone = local.zone
26+
}
27+
28+
provider "ibm" {
29+
alias = "vpc"
30+
ibmcloud_api_key = local.key
31+
region = "eu-de"
32+
}

0 commit comments

Comments
 (0)