diff --git a/.gitignore b/.gitignore index 8c6f14d63abe07809ad3176aa256ddc714d848a7..a2b9a4ab5f71e40640062189935a80460ece8a45 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ salt/pillar/environments/** !salt/pillar/environments/local_cloud4/** +!salt/pillar/environments/ci_cloud4/** ### GENERIC STUFF: @@ -384,6 +385,7 @@ scrubfile.json .terraform* **/.terraform **/terraform.tfstate +**/terraform.tfvars # VS Code .vscode/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d072b91c8f4874f3564fdcae5179d6bb56eb7d5e..5afb9ccb9a0e192a3dce3ccc183e067690e96ec1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,4 @@ include: - 'documentation/pipeline.yml' - - 'salt/pipeline.yml' \ No newline at end of file + - 'salt/pipeline.yml' + - 'terraform/ci.yml' \ No newline at end of file diff --git a/documentation/docs/20_29_high_level_overview/24_bare_metal.md b/documentation/docs/20_29_high_level_overview/24_bare_metal.md new file mode 100644 index 0000000000000000000000000000000000000000..729c567c6bf5d3eb5cac8cf0010c1dfff8116087 --- /dev/null +++ b/documentation/docs/20_29_high_level_overview/24_bare_metal.md @@ -0,0 +1,4 @@ +Bare Metal in Openstack +======================= + + diff --git a/documentation/pipeline.yml b/documentation/pipeline.yml index a0cf5a90a6fe83163aaf0c3d60524a3193f4dc79..34dcd86f2eac793076ae5accfce20dc07816411d 100644 --- a/documentation/pipeline.yml +++ b/documentation/pipeline.yml @@ -3,7 +3,7 @@ image: python:3.10-buster before_script: test_docs: - stage: test + stage: .pre script: - cd documentation - pip install -r requirements.txt diff --git a/salt/pillar/defaults.sls b/salt/pillar/defaults.sls index fc19fa19ed99b90b6859f67d160d6c0868afc91a..cbd37bcd5ab13e1e1cc94da758ce1ab16142daaf 100644 --- a/salt/pillar/defaults.sls +++ b/salt/pillar/defaults.sls @@ -41,8 +41,7 @@ cloud4: mine_functions: - network.ip_addrs: - cidr: 10.10.0.0/24 + network.ip_addrs: [] mysql.host: openstack-database-0 mysql.port: 3306 @@ -170,7 +169,7 @@ loki: openstack: region: "cloud4" - release_version: dalmation + release_version: dalmatian keystone: admin_password: &keystone_admin_pass "Knock.Knock.Whos.There? ... Keystone" diff --git a/salt/pillar/environments/ci_cloud4/defaults.sls b/salt/pillar/environments/ci_cloud4/defaults.sls new file mode 100644 index 0000000000000000000000000000000000000000..4c13c18101d7fe81003ec1c3f73b075ed116bc13 --- /dev/null +++ b/salt/pillar/environments/ci_cloud4/defaults.sls @@ -0,0 +1,38 @@ +motd: + art: |-2 + ██████ ██ ██ ██████ ██ + ██ ██ ██ ██ ██ + ██ ███████ ██ ██ + ██ ██ ██ ██ + ██████ ██ ██████ ██ + + +# Only mine local IP addresses +# 10.10.0.0/24 is considered "public" for the CI +mine_functions: + network.ip_addrs: + - cidr: 10.11.0.0/24 # private network + +ceph: + config: + global: + fsid: "c4c4c4c4-c4c4-c4c4-c4c4c4c4c4" + public_network: 10.11.0.0/24 + cluster_network: 10.12.0.0/24 + osd_disks: + - name: /dev/vdb + type: lvm + +hostsfile: + identity.local.cloud4: + - 10.11.0.100 + image.local.cloud4: + - 10.11.0.100 + network.local.cloud4: + - 10.11.0.100 + compute.local.cloud4: + - 10.11.0.100 + block-storage.local.cloud4: + - 10.11.0.100 + placement.local.cloud4: + - 10.11.0.100 diff --git a/salt/pipeline.yml b/salt/pipeline.yml index 8414fb665a0ec24e46f3f5d4975d9e6eeda36741..4dd4ecbedf915345bdbbd09f401d7ed47a5eb44a 100644 --- a/salt/pipeline.yml +++ b/salt/pipeline.yml @@ -1,6 +1,6 @@ salt-lint: image: python:3.10-buster - stage: test + stage: .pre script: - pip install salt-lint - 'find salt -type f -name "*.sls" -print0 | xargs -0 --no-run-if-empty salt-lint' diff --git a/salt/salt/ceph/config.sls b/salt/salt/ceph/config.sls index b6ab54f7ff7f56d0a30b252de958b5e85b4ab47b..c6d405dec81fc84dc2b607c2b3f493acdb7b564c 100644 --- a/salt/salt/ceph/config.sls +++ b/salt/salt/ceph/config.sls @@ -2,7 +2,7 @@ # vim: set ft=python ts=4 sw=4 et: import collections.abc - +from ipaddress import ip_network, ip_address def run(): """ @@ -23,14 +23,22 @@ def run(): mon_hosts = [] mon_initial_members = [] mds_config = {} + + cluster_network = ip_network(__salt__['pillar.get']("ceph:config:global:cluster_network")) + for minion_name, addrs in __salt__["mine.get"]( "ceph-monitor*", "network.ip_addrs" ).items(): - mon_initial_members.append(minion_name) - mon_hosts.extend(addrs) # add all IP addresses of the monitor - # Each mon also has an mds, so we need to add the mds to the config - mds_config[f"mds.{minion_name}"] = {"host": minion_name} - + # Check every address for this minion + for addr in addrs: + ip = ip_address(addr) + # Check if the IP address is in the cluster network + if ip in cluster_network: + # If it is, we add it to the list of monitors + mon_initial_members.append(minion_name) + mon_hosts.append(addr) + mds_config[f"mds.{minion_name}"] = {"host": minion_name} + break # We've found the address for this minion, so we can stop looking # Add the just assembled list of monitors to the configuration. config_items = { diff --git a/terraform/ci.yml b/terraform/ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..5106d719e156c135c5091065acf3743fadb76edb --- /dev/null +++ b/terraform/ci.yml @@ -0,0 +1,121 @@ +stages: + - validate + - build + - deploy + - test + - destroy + +variables: + TF_STATE_NAME: merlin-cloud4-ci + TF_ROOT: terraform/openstack + TF_VAR_external_network_name: vlan1066 + TF_VAR_base_image_name: Rocky-9.5 + TF_VAR_default_small_flavor_name: m1.medium + TF_VAR_deploy_username: rocky + TF_VAR_deploy_environment_name: "ci.cloud4" + TF_VAR_ssh_private_key_file: .terraform.key + TF_VAR_ssh_public_key_file: .terraform.key.pub + +cache: + key: ${TF_STATE_NAME} + paths: + - terraform/openstack/.terraform/ + - terraform/openstack/.terraform.key + - terraform/openstack/.terraform.key.pub + +terraform_init: + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + stage: .pre + script: + - echo "$TF_ROOT" + - cd ${TF_ROOT} + - "if [ ! -f .terraform.key ]; then ssh-keygen -t ed25519 -f .terraform.key -N ''; fi" + - gitlab-terraform init + only: + - main + - branches + - tags + - merge_requests + tags: + - container + +terraform_fmt: + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + stage: .pre + script: + - echo "$TF_ROOT" + - cd ${TF_ROOT} + - gitlab-terraform fmt + only: + - main + - branches + - tags + - merge_requests + tags: + - container + +terraform_validate: + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + stage: validate + script: + - echo "$TF_ROOT" + - cd ${TF_ROOT} + - gitlab-terraform validate + only: + - main + - branches + - tags + - merge_requests + tags: + - container + +# terraform_plan: +# image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest +# stage: build +# script: +# - echo "$TF_ROOT" +# - cd ${TF_ROOT} +# - gitlab-terraform plan +# - gitlab-terraform plan-json +# artifacts: +# name: plan +# paths: +# - ${TF_ROOT}/plan.cache +# only: +# - main +# - branches +# - tags +# - merge_requests +# tags: +# - container + +terraform_apply: + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + stage: deploy + script: + - echo "$TF_ROOT" + - cd ${TF_ROOT} + - gitlab-terraform plan + - gitlab-terraform apply +# dependencies: +# - terraform_plan + only: + - main + - merge_requests + tags: + - container + +terraform_destroy: + image: registry.gitlab.com/gitlab-org/terraform-images/stable:latest + stage: destroy + script: + - echo "$TF_ROOT" + - cd ${TF_ROOT} + - gitlab-terraform destroy + dependencies: + - terraform_apply + only: + - main + - merge_requests + tags: + - container diff --git a/terraform/cephmon.tf b/terraform/local_libvirt/cephmon.tf similarity index 100% rename from terraform/cephmon.tf rename to terraform/local_libvirt/cephmon.tf diff --git a/terraform/cephosd.tf b/terraform/local_libvirt/cephosd.tf similarity index 100% rename from terraform/cephosd.tf rename to terraform/local_libvirt/cephosd.tf diff --git a/terraform/cloud_init/meta_data.tftpl b/terraform/local_libvirt/cloud_init/meta_data.tftpl similarity index 100% rename from terraform/cloud_init/meta_data.tftpl rename to terraform/local_libvirt/cloud_init/meta_data.tftpl diff --git a/terraform/cloud_init/network_config.tftpl b/terraform/local_libvirt/cloud_init/network_config.tftpl similarity index 100% rename from terraform/cloud_init/network_config.tftpl rename to terraform/local_libvirt/cloud_init/network_config.tftpl diff --git a/terraform/cloud_init/user_data.tftpl b/terraform/local_libvirt/cloud_init/user_data.tftpl similarity index 87% rename from terraform/cloud_init/user_data.tftpl rename to terraform/local_libvirt/cloud_init/user_data.tftpl index e114ba9c19ffb3f5633558c9f0ca8fdccaa3be3e..fae1c617e2b71ff92eb07ef3d081d1045c6fd2bb 100644 --- a/terraform/cloud_init/user_data.tftpl +++ b/terraform/local_libvirt/cloud_init/user_data.tftpl @@ -2,6 +2,11 @@ #host_type=${host_type} +%{ if set_hostname ~} +hostname: ${hostname} +fqdn: ${hostname}.${deploy_environment_name} +%{ endif ~} + password: banaan chpasswd: expire: False @@ -12,8 +17,9 @@ ssh_authorized_keys: runcmd: - "set -x" %{ if host_type != "salt-master" ~} - - "echo '10.10.0.2 salt' >> /etc/hosts" + - "sudo echo '${salt_master_ip} salt' >> /etc/hosts" %{ endif ~} + - "sudo echo '127.0.0.1 ${hostname}' >> /etc/hosts" - 'sudo dnf install --assumeyes --refresh epel-release' - 'sudo dnf clean all' - 'sudo rm -Rf /var/cache/dnf/' diff --git a/terraform/cloudinit.tf b/terraform/local_libvirt/cloudinit.tf similarity index 86% rename from terraform/cloudinit.tf rename to terraform/local_libvirt/cloudinit.tf index 569247978b2ec8113109db5ec8b030135564c753..612a0d115cffe0e25517fa6d3e8eeb10fb7a29c8 100644 --- a/terraform/cloudinit.tf +++ b/terraform/local_libvirt/cloudinit.tf @@ -20,6 +20,9 @@ data "template_file" "salt_user_data" { host_type = "salt-master" salt_version = var.salt_version ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = "10.10.0.2" + set_hostname = false + hostname = "" } } @@ -31,6 +34,9 @@ data "template_file" "salt_minion_user_data" { host_type = "salt-minion" salt_version = var.salt_version ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = libvirt_domain.salt.network_interface.0.addresses.0 + set_hostname = false + hostname = "" } } @@ -42,6 +48,9 @@ data "template_file" "salt_minion_openstack_user_data" { host_type = "salt-minion-openstack" salt_version = var.salt_version ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = libvirt_domain.salt.network_interface.0.addresses.0 + set_hostname = false + hostname = "" } } diff --git a/terraform/compute.tf b/terraform/local_libvirt/compute.tf similarity index 100% rename from terraform/compute.tf rename to terraform/local_libvirt/compute.tf diff --git a/terraform/controller.tf b/terraform/local_libvirt/controller.tf similarity index 100% rename from terraform/controller.tf rename to terraform/local_libvirt/controller.tf diff --git a/terraform/database.tf b/terraform/local_libvirt/database.tf similarity index 100% rename from terraform/database.tf rename to terraform/local_libvirt/database.tf diff --git a/terraform/graph.svg b/terraform/local_libvirt/graph.svg similarity index 100% rename from terraform/graph.svg rename to terraform/local_libvirt/graph.svg diff --git a/terraform/monitor.tf b/terraform/local_libvirt/monitor.tf similarity index 100% rename from terraform/monitor.tf rename to terraform/local_libvirt/monitor.tf diff --git a/terraform/network.tf b/terraform/local_libvirt/network.tf similarity index 100% rename from terraform/network.tf rename to terraform/local_libvirt/network.tf diff --git a/terraform/networker.tf b/terraform/local_libvirt/networker.tf similarity index 100% rename from terraform/networker.tf rename to terraform/local_libvirt/networker.tf diff --git a/terraform/salt.tf b/terraform/local_libvirt/salt.tf similarity index 100% rename from terraform/salt.tf rename to terraform/local_libvirt/salt.tf diff --git a/terraform/scripts/.gitkeep b/terraform/local_libvirt/scripts/.gitkeep similarity index 100% rename from terraform/scripts/.gitkeep rename to terraform/local_libvirt/scripts/.gitkeep diff --git a/terraform/serviceproxy.tf b/terraform/local_libvirt/serviceproxy.tf similarity index 100% rename from terraform/serviceproxy.tf rename to terraform/local_libvirt/serviceproxy.tf diff --git a/terraform/terraform.tf b/terraform/local_libvirt/terraform.tf similarity index 100% rename from terraform/terraform.tf rename to terraform/local_libvirt/terraform.tf diff --git a/terraform/openstack/ceph_monitor.tf b/terraform/openstack/ceph_monitor.tf new file mode 100644 index 0000000000000000000000000000000000000000..50f470548f85b67200b4f0fa7d002316c019f4bd --- /dev/null +++ b/terraform/openstack/ceph_monitor.tf @@ -0,0 +1,90 @@ +variable "ceph_mon_count" { + type = number + default = 1 +} + +variable "ceph_mon_size" { + type = number + default = 50 +} + +resource "openstack_networking_port_v2" "ceph_monitor_private" { + count = var.ceph_mon_count + network_id = openstack_networking_network_v2.private.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.private.id + ip_address = cidrhost(openstack_networking_subnet_v2.private.cidr, count.index + 80) + } +} + +resource "openstack_networking_port_v2" "ceph_monitor_storage" { + count = var.ceph_mon_count + network_id = openstack_networking_network_v2.storage.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.storage.id + ip_address = cidrhost(openstack_networking_subnet_v2.storage.cidr, count.index + 80) + } +} + +resource "openstack_blockstorage_volume_v3" "ceph_monitor_persistent_volume" { + count = var.ceph_mon_count + size = 50 + image_id = data.openstack_images_image_v2.base_image.id + enable_online_resize = true + depends_on = [ + openstack_networking_floatingip_assoaciate_v2.salt + ] +} + +resource "openstack_compute_instance_v2" "ceph_monitor" { + count = var.ceph_mon_count + name = "ceph-monitor-${count.index}.${var.deploy_environment_name}" + + flavor_id = data.openstack_compute_flavor_v2.small.id + + user_data = templatefile("${path.module}/cloud_init/user_data.tftpl", { + deploy_environment_name = var.deploy_environment_name + deploy_username = var.deploy_username + host_type = "ceph-monitor" + salt_version = var.salt_version + ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = openstack_networking_port_v2.salt_private.fixed_ip[0].ip_address + set_hostname = true + hostname = "ceph-monitor-${count.index}" + }) + + connection { + type = "ssh" + user = var.deploy_username + host = self.access_ip_v4 + private_key = file(var.ssh_private_key_file) + + bastion_host = openstack_compute_floatingip_v2.salt.address + bastion_user = var.deploy_username + bastion_private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "until [ -f /run/cloud-init-done ]; do sleep 5; done" + ] + } + + network { + port = element(openstack_networking_port_v2.ceph_monitor_private.*.id, count.index) + } + + network { + port = element(openstack_networking_port_v2.ceph_monitor_storage.*.id, count.index) + } + + block_device { + source_type = "volume" + destination_type = "volume" + uuid = element(openstack_blockstorage_volume_v3.ceph_monitor_persistent_volume.*.id, count.index) + } +} diff --git a/terraform/openstack/ceph_osd.tf b/terraform/openstack/ceph_osd.tf new file mode 100644 index 0000000000000000000000000000000000000000..00cd9885334a314fba81ddd56cb64f3f59e54b88 --- /dev/null +++ b/terraform/openstack/ceph_osd.tf @@ -0,0 +1,102 @@ +variable "ceph_osd_count" { + type = number + default = 3 +} + +variable "ceph_osd_size" { + type = number + default = 100 +} + +resource "openstack_networking_port_v2" "ceph_osd_private" { + count = var.ceph_osd_count + network_id = openstack_networking_network_v2.private.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.private.id + ip_address = cidrhost(openstack_networking_subnet_v2.private.cidr, count.index + 70) + } +} + +resource "openstack_networking_port_v2" "ceph_osd_storage" { + count = var.ceph_osd_count + network_id = openstack_networking_network_v2.storage.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.storage.id + ip_address = cidrhost(openstack_networking_subnet_v2.storage.cidr, count.index + 70) + } +} + +resource "openstack_blockstorage_volume_v3" "ceph_osd_persistent_volume" { + count = var.ceph_osd_count + size = var.ceph_osd_size + image_id = data.openstack_images_image_v2.base_image.id + depends_on = [ + openstack_networking_floatingip_assoaciate_v2.salt + ] +} + +resource "openstack_blockstorage_volume_v3" "ceph_osd_data_disk" { + size = 100 + count = var.ceph_osd_count +} + +resource "openstack_compute_instance_v2" "ceph_osd" { + count = var.ceph_osd_count + name = "ceph-osd-${count.index}.${var.deploy_environment_name}" + + flavor_id = data.openstack_compute_flavor_v2.small.id + + user_data = templatefile("${path.module}/cloud_init/user_data.tftpl", { + deploy_environment_name = var.deploy_environment_name + deploy_username = var.deploy_username + host_type = "ceph-osd" + salt_version = var.salt_version + ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = openstack_networking_port_v2.salt_private.fixed_ip[0].ip_address + set_hostname = true + hostname = "ceph-osd-${count.index}" + }) + + connection { + type = "ssh" + user = var.deploy_username + host = self.access_ip_v4 + private_key = file(var.ssh_private_key_file) + + bastion_host = openstack_compute_floatingip_v2.salt.address + bastion_user = var.deploy_username + bastion_private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "until [ -f /run/cloud-init-done ]; do sleep 5; done" + ] + } + + network { + port = openstack_networking_port_v2.ceph_osd_private.*.id[count.index] + } + + network { + port = openstack_networking_port_v2.ceph_osd_storage.*.id[count.index] + } + + block_device { + source_type = "volume" + destination_type = "volume" + uuid = openstack_blockstorage_volume_v3.ceph_osd_persistent_volume.*.id[count.index] + boot_index = 0 + } +} + +resource "openstack_compute_volume_attach_v2" "ceph_osd_data_disk" { + count = var.ceph_osd_count + instance_id = openstack_compute_instance_v2.ceph_osd.*.id[count.index] + volume_id = openstack_blockstorage_volume_v3.ceph_osd_data_disk.*.id[count.index] + device = "/dev/vdb" +} diff --git a/terraform/openstack/cloud_init b/terraform/openstack/cloud_init new file mode 120000 index 0000000000000000000000000000000000000000..53fa83e3f19903ab4216ab72c5c95908fbb303d3 --- /dev/null +++ b/terraform/openstack/cloud_init @@ -0,0 +1 @@ +../local_libvirt/cloud_init \ No newline at end of file diff --git a/terraform/openstack/flavor.tf b/terraform/openstack/flavor.tf new file mode 100644 index 0000000000000000000000000000000000000000..64305bf5ce7a761e0b2a0a75856edb7b319a5a94 --- /dev/null +++ b/terraform/openstack/flavor.tf @@ -0,0 +1,3 @@ +data "openstack_compute_flavor_v2" "small" { + name = var.default_small_flavor_name +} \ No newline at end of file diff --git a/terraform/openstack/image.tf b/terraform/openstack/image.tf new file mode 100644 index 0000000000000000000000000000000000000000..5335d550e2b768757df50f422a9288cbdaed8435 --- /dev/null +++ b/terraform/openstack/image.tf @@ -0,0 +1,3 @@ +data "openstack_images_image_v2" "base_image" { + name = var.base_image_name +} \ No newline at end of file diff --git a/terraform/openstack/monitor.tf b/terraform/openstack/monitor.tf new file mode 100644 index 0000000000000000000000000000000000000000..f0b7fc2f5ec215ceb14ab7e27927f5a75282d43f --- /dev/null +++ b/terraform/openstack/monitor.tf @@ -0,0 +1,91 @@ +variable "monitor_count" { + type = number + default = 1 +} + +variable "monitor_size" { + type = number + default = 50 +} + +resource "openstack_networking_port_v2" "monitor_private" { + count = var.monitor_count + network_id = openstack_networking_network_v2.private.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.private.id + ip_address = cidrhost(openstack_networking_subnet_v2.private.cidr, count.index + 5) + } +} + +resource "openstack_networking_port_v2" "monitor_public" { + count = var.monitor_count + network_id = openstack_networking_network_v2.public.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.public.id + ip_address = cidrhost(openstack_networking_subnet_v2.public.cidr, count.index + 5) + } +} + + +resource "openstack_blockstorage_volume_v3" "monitor_persistent_volume" { + image_id = data.openstack_images_image_v2.base_image.id + count = var.monitor_count + size = var.monitor_size +} + +resource "openstack_compute_instance_v2" "monitor" { + count = var.monitor_count + name = "openstack-serviceproxy-${count.index}.${var.deploy_environment_name}" + + flavor_id = data.openstack_compute_flavor_v2.small.id + + user_data = templatefile("${path.module}/cloud_init/user_data.tftpl", { + deploy_environment_name = var.deploy_environment_name + deploy_username = var.deploy_username + host_type = "salt-minion" + salt_version = var.salt_version + ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = openstack_networking_port_v2.salt_private.fixed_ip[0].ip_address + set_hostname = true + hostname = "monitoring-${count.index}" + }) + + connection { + type = "ssh" + user = var.deploy_username + host = self.access_ip_v4 + private_key = file(var.ssh_private_key_file) + + bastion_host = openstack_compute_floatingip_v2.salt.address + bastion_user = var.deploy_username + bastion_private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "until [ -f /run/cloud-init-done ]; do sleep 5; done" + ] + } + + network { + port = element(openstack_networking_port_v2.monitor_public.*.id, count.index) + } + + network { + port = element(openstack_networking_port_v2.monitor_private.*.id, count.index) + } + + block_device { + source_type = "volume" + destination_type = "volume" + uuid = element(openstack_blockstorage_volume_v3.monitor_persistent_volume.*.id, count.index) + boot_index = 0 + } +} + + + diff --git a/terraform/openstack/networking.tf b/terraform/openstack/networking.tf new file mode 100644 index 0000000000000000000000000000000000000000..1fd87e2d6246834a902edb12b589a04770da8918 --- /dev/null +++ b/terraform/openstack/networking.tf @@ -0,0 +1,66 @@ +data "openstack_networking_network_v2" "external" { + name = var.external_network_name +} +resource "openstack_networking_network_v2" "public" { + name = "cloud4_public_network" + admin_state_up = "true" + mtu = 1450 +} +resource "openstack_networking_subnet_v2" "public" { + name = "cloud4_public_subnet" + network_id = openstack_networking_network_v2.public.id + cidr = "10.10.0.0/24" + ip_version = 4 + dns_nameservers = ["1.1.1.1", "1.0.0.1"] +} +resource "openstack_networking_router_v2" "os_router" { + name = "cloud4_public_router" + admin_state_up = true + external_network_id = data.openstack_networking_network_v2.external.id +} +resource "openstack_networking_router_interface_v2" "public_subnet_interface" { + router_id = openstack_networking_router_v2.os_router.id + subnet_id = openstack_networking_subnet_v2.public.id +} + +resource "openstack_networking_router_interface_v2" "private_subnet_interface" { + router_id = openstack_networking_router_v2.os_router.id + subnet_id = openstack_networking_subnet_v2.private.id +} + +resource "openstack_networking_network_v2" "private" { + name = "cloud4_private_network" + admin_state_up = "true" + mtu = 1450 +} +resource "openstack_networking_subnet_v2" "private" { + name = "cloud4_private_subnet" + network_id = openstack_networking_network_v2.private.id + cidr = "10.11.0.0/24" + ip_version = 4 + + enable_dhcp = true +} +resource "openstack_networking_floatingip_v2" "reserved" { + count = 2 + pool = var.external_network_name + lifecycle { + # Set this to true to keep hold of the floating IPs when the environment is destroyed + prevent_destroy = false + } +} + +resource "openstack_networking_network_v2" "storage" { + name = "cloud4_storage_network" + admin_state_up = "true" + mtu = 8950 +} + +resource "openstack_networking_subnet_v2" "storage" { + name = "cloud4_storage_subnet" + network_id = openstack_networking_network_v2.storage.id + cidr = "10.12.0.0/24" + ip_version = 4 + + enable_dhcp = true +} diff --git a/terraform/openstack/openstack_compute.tf b/terraform/openstack/openstack_compute.tf new file mode 100644 index 0000000000000000000000000000000000000000..f7fc6e2b1a7578a89cb40ec35453442d8b82285b --- /dev/null +++ b/terraform/openstack/openstack_compute.tf @@ -0,0 +1,91 @@ +variable "openstack_compute_count" { + type = number + default = 2 +} + +variable "openstack_compute_size" { + type = number + default = 100 +} + +resource "openstack_networking_port_v2" "openstack_compute_private" { + count = var.openstack_compute_count + network_id = openstack_networking_network_v2.private.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.private.id + ip_address = cidrhost(openstack_networking_subnet_v2.private.cidr, count.index + 20) + } +} + +resource "openstack_networking_port_v2" "openstack_compute_storage" { + count = var.openstack_compute_count + network_id = openstack_networking_network_v2.storage.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.storage.id + ip_address = cidrhost(openstack_networking_subnet_v2.storage.cidr, count.index + 20) + } +} + +resource "openstack_blockstorage_volume_v3" "openstack_compute_persistent_volume" { + count = var.openstack_compute_count + size = var.openstack_compute_size + image_id = data.openstack_images_image_v2.base_image.id + + depends_on = [ + openstack_networking_floatingip_assoaciate_v2.salt + ] +} + +resource "openstack_compute_instance_v2" "openstack_compute" { + count = var.openstack_compute_count + name = "openstack-compute-${count.index}.${var.deploy_environment_name}" + + flavor_id = data.openstack_compute_flavor_v2.small.id + + user_data = templatefile("${path.module}/cloud_init/user_data.tftpl", { + deploy_environment_name = var.deploy_environment_name + deploy_username = var.deploy_username + host_type = "salt-minion-openstack" + salt_version = var.salt_version + ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = openstack_networking_port_v2.salt_private.fixed_ip[0].ip_address + set_hostname = true + hostname = "openstack-compute-${count.index}" + }) + + connection { + type = "ssh" + user = var.deploy_username + host = self.access_ip_v4 + private_key = file(var.ssh_private_key_file) + + bastion_host = openstack_compute_floatingip_v2.salt.address + bastion_user = var.deploy_username + bastion_private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "until [ -f /run/cloud-init-done ]; do sleep 5; done" + ] + } + + network { + port = element(openstack_networking_port_v2.openstack_compute_private.*.id, count.index) + } + + network { + port = element(openstack_networking_port_v2.openstack_compute_storage.*.id, count.index) + } + + block_device { + source_type = "volume" + destination_type = "volume" + uuid = element(openstack_blockstorage_volume_v3.openstack_compute_persistent_volume.*.id, count.index) + boot_index = 0 + } +} diff --git a/terraform/openstack/openstack_controller.tf b/terraform/openstack/openstack_controller.tf new file mode 100644 index 0000000000000000000000000000000000000000..6839857afac5f7e008c5ee10decefa255d6fdee6 --- /dev/null +++ b/terraform/openstack/openstack_controller.tf @@ -0,0 +1,89 @@ +variable "openstack_controller_count" { + type = number + default = 2 +} + +variable "openstack_controller_size" { + type = number + default = 20 +} + +resource "openstack_networking_port_v2" "openstack_controller_private" { + count = var.openstack_controller_count + network_id = openstack_networking_network_v2.private.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.private.id + ip_address = cidrhost(openstack_networking_subnet_v2.private.cidr, count.index + 10) + } +} + +resource "openstack_networking_port_v2" "openstack_controller_storage" { + count = var.openstack_controller_count + network_id = openstack_networking_network_v2.storage.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.storage.id + ip_address = cidrhost(openstack_networking_subnet_v2.storage.cidr, count.index + 10) + } +} + +resource "openstack_blockstorage_volume_v3" "openstack_controller_persistent_volume" { + count = var.openstack_controller_count + size = var.openstack_controller_size + image_id = data.openstack_images_image_v2.base_image.id + depends_on = [ + openstack_networking_floatingip_assoaciate_v2.salt + ] +} + +resource "openstack_compute_instance_v2" "openstack_controller" { + count = var.openstack_controller_count + name = "openstack-controller-${count.index}.${var.deploy_environment_name}" + + flavor_id = data.openstack_compute_flavor_v2.small.id + + user_data = templatefile("${path.module}/cloud_init/user_data.tftpl", { + deploy_environment_name = var.deploy_environment_name + deploy_username = var.deploy_username + host_type = "salt-minion-openstack" + salt_version = var.salt_version + ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = openstack_networking_port_v2.salt_private.fixed_ip[0].ip_address + set_hostname = true + hostname = "openstack-controller-${count.index}" + }) + + connection { + type = "ssh" + user = var.deploy_username + host = self.access_ip_v4 + private_key = file(var.ssh_private_key_file) + + bastion_host = openstack_compute_floatingip_v2.salt.address + bastion_user = var.deploy_username + bastion_private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "until [ -f /run/cloud-init-done ]; do sleep 5; done" + ] + } + + network { + port = element(openstack_networking_port_v2.openstack_controller_private.*.id, count.index) + } + + network { + port = element(openstack_networking_port_v2.openstack_controller_storage.*.id, count.index) + } + + block_device { + source_type = "volume" + destination_type = "volume" + uuid = element(openstack_blockstorage_volume_v3.openstack_controller_persistent_volume.*.id, count.index) + } +} diff --git a/terraform/openstack/openstack_database.tf b/terraform/openstack/openstack_database.tf new file mode 100644 index 0000000000000000000000000000000000000000..1a0882195794220f49e99fa8f44a61a395f19dc4 --- /dev/null +++ b/terraform/openstack/openstack_database.tf @@ -0,0 +1,74 @@ +variable "openstack_database_count" { + type = number + default = 2 +} + +variable "openstack_database_size" { + type = number + default = 20 +} + +resource "openstack_networking_port_v2" "openstack_database_private" { + count = var.openstack_database_count + network_id = openstack_networking_network_v2.private.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.private.id + ip_address = cidrhost(openstack_networking_subnet_v2.private.cidr, count.index + 30) + } +} + +resource "openstack_blockstorage_volume_v3" "openstack_database_persistent_volume" { + count = var.openstack_database_count + size = var.openstack_database_size + image_id = data.openstack_images_image_v2.base_image.id + depends_on = [ + openstack_networking_floatingip_assoaciate_v2.salt + ] +} + +resource "openstack_compute_instance_v2" "openstack_database" { + count = var.openstack_database_count + name = "openstack-database-${count.index}.${var.deploy_environment_name}" + + flavor_id = data.openstack_compute_flavor_v2.small.id + + user_data = templatefile("${path.module}/cloud_init/user_data.tftpl", { + deploy_environment_name = var.deploy_environment_name + deploy_username = var.deploy_username + host_type = "salt-minion-openstack" + salt_version = var.salt_version + ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = openstack_networking_port_v2.salt_private.fixed_ip[0].ip_address + set_hostname = true + hostname = "openstack-database-${count.index}" + }) + + connection { + type = "ssh" + user = var.deploy_username + host = self.access_ip_v4 + private_key = file(var.ssh_private_key_file) + + bastion_host = openstack_compute_floatingip_v2.salt.address + bastion_user = var.deploy_username + bastion_private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "until [ -f /run/cloud-init-done ]; do sleep 5; done" + ] + } + + network { + port = element(openstack_networking_port_v2.openstack_database_private.*.id, count.index) + } + + block_device { + source_type = "volume" + destination_type = "volume" + uuid = element(openstack_blockstorage_volume_v3.openstack_database_persistent_volume.*.id, count.index) + } +} diff --git a/terraform/openstack/openstack_networker.tf b/terraform/openstack/openstack_networker.tf new file mode 100644 index 0000000000000000000000000000000000000000..6637fb0326f7ecfe2e0a853f66be0e44ac50140f --- /dev/null +++ b/terraform/openstack/openstack_networker.tf @@ -0,0 +1,84 @@ +variable "openstack_networker_count" { + type = number + default = 1 +} + +resource "openstack_networking_port_v2" "openstack_networker_private" { + count = var.openstack_networker_count + network_id = openstack_networking_network_v2.private.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.private.id + ip_address = cidrhost(openstack_networking_subnet_v2.private.cidr, count.index + 40) + } +} + +resource "openstack_networking_port_v2" "openstack_networker_public" { + count = var.openstack_networker_count + network_id = openstack_networking_network_v2.public.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.public.id + ip_address = cidrhost(openstack_networking_subnet_v2.public.cidr, count.index + 40) + } +} + +resource "openstack_blockstorage_volume_v3" "networker_persistent_volume" { + count = var.openstack_networker_count + size = 30 + image_id = data.openstack_images_image_v2.base_image.id + depends_on = [ + openstack_networking_floatingip_assoaciate_v2.salt + ] +} + +resource "openstack_compute_instance_v2" "openstack_networker" { + count = var.openstack_networker_count + name = "openstack-networker-${count.index}.${var.deploy_environment_name}" + + flavor_id = data.openstack_compute_flavor_v2.small.id + + user_data = templatefile("${path.module}/cloud_init/user_data.tftpl", { + deploy_environment_name = var.deploy_environment_name + deploy_username = var.deploy_username + host_type = "salt-minion-openstack" + salt_version = var.salt_version + ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = openstack_networking_port_v2.salt_private.fixed_ip[0].ip_address + set_hostname = true + hostname = "openstack-networker-${count.index}" + }) + + connection { + type = "ssh" + user = var.deploy_username + host = self.access_ip_v4 + private_key = file(var.ssh_private_key_file) + + bastion_host = openstack_compute_floatingip_v2.salt.address + bastion_user = var.deploy_username + bastion_private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "until [ -f /run/cloud-init-done ]; do sleep 5; done" + ] + } + + network { + port = element(openstack_networking_port_v2.openstack_networker_public.*.id, count.index) + } + + network { + port = element(openstack_networking_port_v2.openstack_networker_private.*.id, count.index) + } + + block_device { + source_type = "volume" + destination_type = "volume" + uuid = element(openstack_blockstorage_volume_v3.networker_persistent_volume.*.id, count.index) + } +} diff --git a/terraform/openstack/openstack_serviceproxy.tf b/terraform/openstack/openstack_serviceproxy.tf new file mode 100644 index 0000000000000000000000000000000000000000..dacb22b568b31dc26b693fdd07f53ee26ce76cbf --- /dev/null +++ b/terraform/openstack/openstack_serviceproxy.tf @@ -0,0 +1,84 @@ +variable "openstack_serviceproxy_count" { + type = number + default = 1 +} + +resource "openstack_networking_port_v2" "openstack_serviceproxy_private" { + count = var.openstack_serviceproxy_count + network_id = openstack_networking_network_v2.private.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.private.id + ip_address = cidrhost(openstack_networking_subnet_v2.private.cidr, count.index + 100) + } +} + +resource "openstack_networking_port_v2" "openstack_serviceproxy_public" { + count = var.openstack_serviceproxy_count + network_id = openstack_networking_network_v2.public.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.public.id + ip_address = cidrhost(openstack_networking_subnet_v2.public.cidr, count.index + 100) + } +} + +resource "openstack_blockstorage_volume_v3" "openstack_serviceproxy_persistent_volume" { + count = var.openstack_serviceproxy_count + size = 40 + image_id = data.openstack_images_image_v2.base_image.id + depends_on = [ + openstack_networking_floatingip_assoaciate_v2.salt + ] +} + +resource "openstack_compute_instance_v2" "openstack_serviceproxy" { + count = var.openstack_serviceproxy_count + name = "openstack-serviceproxy-${count.index}.${var.deploy_environment_name}" + + flavor_id = data.openstack_compute_flavor_v2.small.id + + user_data = templatefile("${path.module}/cloud_init/user_data.tftpl", { + deploy_environment_name = var.deploy_environment_name + deploy_username = var.deploy_username + host_type = "salt-minion-openstack" + salt_version = var.salt_version + ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = openstack_networking_port_v2.salt_private.fixed_ip[0].ip_address + set_hostname = true + hostname = "openstack-serviceproxy-${count.index}" + }) + + connection { + type = "ssh" + user = var.deploy_username + host = self.access_ip_v4 + private_key = file(var.ssh_private_key_file) + + bastion_host = openstack_compute_floatingip_v2.salt.address + bastion_user = var.deploy_username + bastion_private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "until [ -f /run/cloud-init-done ]; do sleep 5; done" + ] + } + + network { + port = element(openstack_networking_port_v2.openstack_serviceproxy_public.*.id, count.index) + } + + network { + port = element(openstack_networking_port_v2.openstack_serviceproxy_private.*.id, count.index) + } + + block_device { + source_type = "volume" + destination_type = "volume" + uuid = element(openstack_blockstorage_volume_v3.openstack_serviceproxy_persistent_volume.*.id, count.index) + } +} diff --git a/terraform/openstack/salt.tf b/terraform/openstack/salt.tf new file mode 100644 index 0000000000000000000000000000000000000000..db65d39f44903052780c9b448cfd14e496caa007 --- /dev/null +++ b/terraform/openstack/salt.tf @@ -0,0 +1,217 @@ +resource "openstack_networking_port_v2" "salt_private" { + name = "salt_private" + network_id = openstack_networking_network_v2.private.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.private.id + ip_address = cidrhost(openstack_networking_subnet_v2.private.cidr, 3) + } +} + +resource "openstack_networking_port_v2" "salt_public" { + name = "salt_public" + network_id = openstack_networking_network_v2.public.id + admin_state_up = true + + fixed_ip { + subnet_id = openstack_networking_subnet_v2.public.id + ip_address = cidrhost(openstack_networking_subnet_v2.public.cidr, 3) + } +} + +resource "openstack_blockstorage_volume_v3" "salt_persistent_volume" { + name = "salt_persistent_volume" + description = "Stores persistent state for the salt master." + size = 20 + enable_online_resize = true + image_id = data.openstack_images_image_v2.base_image.id +} + +data "openstack_networking_secgroup_v2" "default" { + name = "default" +} + +resource "openstack_networking_secgroup_rule_v2" "salt_ssh_in" { + direction = "ingress" + security_group_id = data.openstack_networking_secgroup_v2.default.id + port_range_min = 22 + port_range_max = 22 + protocol = "tcp" + ethertype = "IPv4" +} + +resource "openstack_compute_floatingip_v2" "salt" { + pool = var.external_network_name +} + +resource "openstack_compute_instance_v2" "salt" { + name = "salt.${var.deploy_environment_name}" + security_groups = [ + "default", + ] + + flavor_id = data.openstack_compute_flavor_v2.small.id + + block_device { + source_type = "volume" + destination_type = "volume" + uuid = openstack_blockstorage_volume_v3.salt_persistent_volume.id + boot_index = 0 + } + + user_data = templatefile("${path.module}/cloud_init/user_data.tftpl", { + deploy_environment_name = var.deploy_environment_name + deploy_username = var.deploy_username + host_type = "salt-master" + salt_version = var.salt_version + ssh_public_key = file(var.ssh_public_key_file) + salt_master_ip = openstack_networking_port_v2.salt_private.fixed_ip[0] + set_hostname = true + hostname = "salt" + }) + + network { + port = openstack_networking_port_v2.salt_public.id + } + network { + port = openstack_networking_port_v2.salt_private.id + } +} + +resource "openstack_networking_floatingip_associate_v2" "salt" { + floating_ip = openstack_compute_floatingip_v2.salt.address + port_id = openstack_networking_port_v2.salt.id + + depends_on = [ + openstack_networking_router_interface_v2.public_subnet_interface + ] + + connection { + type = "ssh" + user = var.deploy_username + host = openstack_compute_floatingip_v2.salt.address + private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "until [ -f /run/cloud-init-done ]; do sleep 5; done", + "sudo bash -c 'echo \"auto_accept: True\" > /etc/salt/master.d/auto_accept.conf'", + "sudo systemctl restart salt-master" + ] + } +} + +resource "null_resource" "sync_salt_states" { + depends_on = [ + openstack_networking_floatingip_assoaciate_v2.salt, + openstack_compute_instance_v2.salt, + openstack_compute_instance_v2.ceph_monitor, + openstack_compute_instance_v2.ceph_osd, + openstack_compute_instance_v2.openstack_controller, + openstack_compute_instance_v2.openstack_compute, + openstack_compute_instance_v2.openstack_networker, + openstack_compute_instance_v2.openstack_serviceproxy, + openstack_compute_instance_v2.openstack_database, + openstack_compute_instance_v2.monitor, + ] + + triggers = { + # Easy way to watch for changes in all files in a directory + # Creates a long string of all file hashes concatenated together, and then hashes that again. + # If one file changes, the end hash also changes, triggering resync of states. + dir_hash = sha256(join("", [ + for filename in fileset("${path.cwd}/../../salt/", "**") : + filesha256("../../salt/${filename}") + ])) + } + + connection { + type = "ssh" + user = var.deploy_username + host = openstack_compute_floatingip_v2.salt.address + private_key = file(var.ssh_private_key_file) + } + + provisioner "file" { + source = "${path.module}/../../salt" + destination = "/tmp/" + } + + provisioner "remote-exec" { + inline = [ + "set -x", + "sudo rm -rf /srv/{salt,pillar,reactor,files}", + "sudo mv /tmp/salt/{salt,pillar,reactor,files} /srv/", + "sudo rm -rf /tmp/salt", + "sudo chown root: -R /srv/{salt,reactor,pillar,files}", + "sleep 5", + "sudo salt \\* test.ping", + "sudo salt-call --output-diff state.apply monitoring.prometheus.node_exporter.master_fetch_archive", + "sudo salt-call --output-diff state.highstate", + "sudo systemctl restart salt-master", + "sleep 5", + ] + } +} + +resource "null_resource" "monitoring_orchestrate" { + depends_on = [ + null_resource.sync_salt_states + ] + + connection { + type = "ssh" + user = var.deploy_username + host = openstack_compute_floatingip_v2.salt.address + private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "sudo salt-run manage.status", + "sudo salt-run --state-output mixed state.orchestrate orch.bootstrap_monitoring", + ] + } +} + +resource "null_resource" "ceph_orchestrate" { + depends_on = [ + null_resource.monitoring_orchestrate + ] + + connection { + type = "ssh" + user = var.deploy_username + host = openstack_compute_floatingip_v2.salt.address + private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "sudo salt-run manage.status", + "sudo salt-run --state-output mixed state.orchestrate orch.bootstrap_ceph", + ] + } +} + +resource "null_resource" "openstack_orchestrate" { + depends_on = [ + null_resource.ceph_orchestrate + ] + + connection { + type = "ssh" + user = var.deploy_username + host = openstack_compute_floatingip_v2.salt.address + private_key = file(var.ssh_private_key_file) + } + + provisioner "remote-exec" { + inline = [ + "sudo salt-run manage.status", + "sudo salt-run --state-output mixed state.orchestrate orch.bootstrap_openstack", + ] + } +} diff --git a/terraform/openstack/terraform.tf b/terraform/openstack/terraform.tf new file mode 100644 index 0000000000000000000000000000000000000000..2d791e6373f6341ec51f7f73def761b743b44d45 --- /dev/null +++ b/terraform/openstack/terraform.tf @@ -0,0 +1,52 @@ +variable "external_network_name" { + type = string + description = "The name of the external network to use" +} +variable "base_image_name" { + type = string + description = "The base image to use when creating instance volumes" +} +variable "default_small_flavor_name" { + type = string + description = "The name of the default flavor to use for small instances" + default = "m1.small" +} +variable "deploy_environment_name" { + type = string + description = "Domain to affix to instance names, used for env-specific variables." + default = "local.cloud4" +} +variable "deploy_username" { + type = string + description = "Default user to use when deploying over ssh. Change when using a different OS." + default = "rocky" +} +variable "salt_version" { + type = string + description = "The version of salt to install on the salt master and minions" + default = "3007" +} +variable "ssh_public_key_file" { + type = string + description = "The path to the public key file to use for ssh access" + default = "~/.ssh/id_rsa.pub" +} +variable "ssh_private_key_file" { + type = string + description = "The path to the private key file to use for ssh access" + default = "~/.ssh/id_rsa" +} + +terraform { + backend "http" { + } + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + version = "1.53.0" + } + } +} +provider "openstack" { + max_retries = 5 +}