Enable bootstrapping in the test deployment jobs

Author: smarterclayton

cluster/ci/data/vars.yaml

@@ -12,9 +12,6 @@ openshift_additional_repos: [
 
 # URLs and certs
 
-provision_prefix: origin-ci-
-gce_network_name: "origin-ci-ocp-network"
-
 openshift_master_cluster_public_hostname: api.ci.openshift.org
 openshift_master_cluster_hostname: internal-master.ci.openshift.org
 public_hosted_zone: ci.openshift.org
@@ -27,72 +24,32 @@ openshift_master_console_port: "8443"
 openshift_master_public_api_url: "https://api.ci.openshift.org"
 openshift_master_public_console_url: "https://api.ci.openshift.org/console"
 openshift_node_port_range: 30000-32000
+openshift_node_open_ports: [{"service":"Router stats port","port":"1936/tcp"},{"service":"Allowed open host ports","port":"9000-10000/tcp"},{"service":"Allowed open host ports","port":"9000-10000/udp"}]
 
-openshift_master_named_certificates: [{"certfile": "{{ inventory_dir }}/ssl.crt", "keyfile": "{{ inventory_dir }}/ssl.key", "names": ["api.ci.openshift.org"]}]
+openshift_master_named_certificates: [{"certfile": "{{ files_dir }}/ssl.crt", "keyfile": "{{ files_dir }}/ssl.key", "names": ["api.ci.openshift.org"]}]
 
 # Authentication and authorization
 
-openshift_master_identity_providers: "{{ (lookup('file', '{{ inventory_dir }}/identity-providers.json' ) | default('{\"items\":[]}') | from_json).get('items') }}"
+openshift_master_identity_providers: "{{ (lookup('file', '{{ files_dir }}/identity-providers.json' ) | default('{\"items\":[]}') | from_json).get('items') }}"
 provision_role_mappings: [{"user": "smarterclayton", "role": "cluster-admin"}]
 
 # Post config setting sizes
 
 openshift_hosted_router_replicas: 1
-openshift_hosted_router_certificate: {"certfile": "{{ inventory_dir }}/router.crt", "keyfile": "{{ inventory_dir }}/router.key", "cafile": "{{ inventory_dir }}/router.cacert"}
+openshift_hosted_router_certificate: {"certfile": "{{ files_dir }}/router.crt", "keyfile": "{{ files_dir }}/router.key", "cafile": "{{ files_dir }}/router.cacert"}
 openshift_hosted_router_create_certificate: False
 openshift_hosted_registry_replicas: 1
 openshift_schedulable: True
 
 # GCE provisioning info
 
-# Project ID and zone settings for Google Cloud
-gce_project_id: openshift-gce-devel
-gce_region_name: us-central1
-gce_zone_name: us-central1-a
-gce_service_account: [email protected]
-gce_service_account_keyfile: "{{ inventory_dir }}/gce.json"
-gce_ssh_private_key: /home/cloud-user/.ssh/google_compute_engine
-
-openshift_hosted_registry_storage_gcs_keyfile: "{{ inventory_dir }}/gcs-registry.json"
+openshift_hosted_registry_storage_gcs_keyfile: "{{ files_dir }}/gcs-registry.json"
 openshift_hosted_registry_storage_kind: object
 openshift_hosted_registry_storage_provider: gcs
 openshift_hosted_registry_storage_gcs_bucket: openshift-gce-devel-origin-ci-registry-bucket
 provision_gce_emptydir_quota: 10Gi
-
-# Control which node group router traffic is targeted at.
-provision_gce_router_network_instance_group: ig-m # default: ig-i
-provision_gce_node_groups:
-- name: master
-  suffix: m
-  tags: ocp-master,preserve
-  machine_type: n1-standard-2
-  boot_disk_size: 150
-  scale: 1
-- name: node
-  suffix: n
-  tags: ocp-node,preserve
-  machine_type: n1-standard-8
-  boot_disk_size: 150
-  scale: 10
-- name: node-flex
-  suffix: nf
-  tags: ocp-node,preserve
-  machine_type: n1-standard-4
-  boot_disk_size: 150
-  scale: 0
-
-
-# An image or image family to pull from
-provision_gce_registered_image: rhel-7
-
 provision_gce_docker_storage_driver: overlay2
 
-# Provide a startup script file to the GCE instances
-provision_gce_startup_script_file: # "startup.sh"
-# Provide userdata to the gce instances
-provision_gce_user_data_file:
-
-rhsub_skip: true
 provision_custom_repositories: [
   {
     "name": "oso-rhui-rhel-server-releases",
@@ -126,10 +83,6 @@ provision_custom_repositories: [
   }
 ]
 
-##############
-# New settings
-##############
-
 openshift_gcp_prefix: origin-ci-
 
 openshift_gcp_project: openshift-gce-devel
@@ -139,7 +92,7 @@ openshift_gcp_zone: us-central1-a
 openshift_gcp_network_name: "{{ openshift_gcp_prefix }}network"
 
 openshift_gcp_iam_service_account: [email protected]
-openshift_gcp_iam_service_account_keyfile: "{{ inventory_dir }}/gce.json"
+openshift_gcp_iam_service_account_keyfile: "{{ files_dir }}/gce.json"
 
 openshift_gcp_ssh_private_key: /home/cloud-user/.ssh/google_compute_engine
 
@@ -148,7 +101,7 @@ openshift_gcp_infra_network_instance_group: ig-m
 openshift_gcp_image: 'rhel-7'
 openshift_gcp_base_image: 'centos-7'
 
-openshift_gcp_registry_bucket_keyfile: "{{ inventory_dir }}/gcs-registry.json"
+openshift_gcp_registry_bucket_keyfile: "{{ files_dir }}/gcs-registry.json"
 openshift_gcp_registry_bucket_name: openshift-gce-devel-origin-ci-registry-bucket
 
 openshift_gcp_node_group_config:

cluster/test-deploy/data/.gitignore

@@ -1,3 +1,4 @@
 *
 !.gitignore
 !vars.yml
+!bootstrap-script.sh

cluster/test-deploy/data/bootstrap-script.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/bootstrap" -H "Metadata-Flavor: Google" )" != "true" ]]; then
+  echo "info: Bootstrap is not enabled for this instance, skipping" 1>&2
+  exit 0
+fi
+
+if ! id=$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-id" -H "Metadata-Flavor: Google" ); then
+  echo "error: Unable to get cluster-id for instance from cluster metadata" 1>&2
+  exit 1
+fi
+
+if ! node_group=$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/node-group" -H "Metadata-Flavor: Google" ); then
+  echo "error: Unable to get node-group for instance from cluster metadata" 1>&2
+  exit 1
+fi
+
+if ! config=$( curl -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/bootstrap-config" -H "Metadata-Flavor: Google" 2>/dev/null ); then
+  while true; do
+    if config=$( curl -f "http://metadata.google.internal/computeMetadata/v1/project/attributes/${id}-bootstrap-config" -H "Metadata-Flavor: Google" 2>/dev/null ); then
+      break
+    fi
+    echo "info: waiting for ${id}-bootstrap-config to become available in cluster metadata ..." 1>&2
+    sleep 5
+  done
+fi
+
+echo "Got bootstrap config from metadata"
+mkdir -p /etc/origin/node
+echo -n "${config}" > /etc/origin/node/bootstrap.kubeconfig
+echo "BOOTSTRAP_CONFIG_NAME=node-config-${node_group}" >> /etc/sysconfig/origin-node
+systemctl enable origin-node
+systemctl start origin-node

cluster/test-deploy/data/vars.yaml

@@ -12,9 +12,6 @@ openshift_additional_repos: [
 
 # URLs and certs
 
-provision_prefix: ci-{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}-
-gce_network_name: "ci-{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}-ocp-network"
-
 openshift_master_cluster_public_hostname: api.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.origin-ci-int-gce.dev.rhcloud.com
 openshift_master_cluster_hostname: internal-api.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.origin-ci-int-gce.dev.rhcloud.com
 wildcard_zone: apps.{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}.origin-ci-int-gce.dev.rhcloud.com
@@ -35,62 +32,19 @@ openshift_node_open_ports: [{"service":"Router stats port","port":"1936/tcp"},{"
 # Authentication and authorization
 
 openshift_master_identity_providers: [{"name": "allow_all", "login": "true", "challenge": "true", "kind": "AllowAllPasswordIdentityProvider"}]
-# provision_role_mappings: [{"user": "admin", "role": "cluster-admin"}]
 
 # Post config setting sizes
 
 openshift_hosted_router_replicas: 1
 openshift_hosted_registry_replicas: 1
 openshift_schedulable: True
 
-# GCE provisioning info
-
-# Project ID and zone settings for Google Cloud
-gce_project_id: openshift-gce-devel-ci
-gce_region_name: us-east1
-gce_zone_name: us-east1-c
-gce_service_account: jenkins-ci-provisioner@openshift-gce-devel.iam.gserviceaccount.com
-gce_service_account_keyfile: "{{ inventory_dir }}/gce.json"
-gce_ssh_private_key: /home/cloud-user/.ssh/google_compute_engine
-
-openshift_hosted_registry_storage_gcs_keyfile: "{{ inventory_dir }}/gce.json"
+openshift_hosted_registry_storage_gcs_keyfile: "{{ files_dir }}/gce.json"
 openshift_hosted_registry_storage_kind: object
 openshift_hosted_registry_storage_provider: gcs
 openshift_hosted_registry_storage_gcs_bucket: origin-ci-test-{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}
-
-# Control which node group router traffic is targeted at.
-provision_gce_router_network_instance_group: ig-m # default: ig-i
-provision_gce_node_groups:
-- name: master
-  suffix: m
-  tags: ocp-master
-  machine_type: n1-standard-2
-  boot_disk_size: 150
-  scale: 1
-- name: node
-  suffix: n
-  tags: ocp-node
-  machine_type: n1-standard-2
-  boot_disk_size: 150
-  scale: 3
-- name: node-flex
-  suffix: nf
-  tags: ocp-node
-  machine_type: n1-standard-2
-  boot_disk_size: 150
-  scale: 0
-
-# An image or image family to pull from
-provision_gce_registered_image: rhel-7
-
 provision_gce_docker_storage_driver: overlay2
 
-# Provide a startup script file to the GCE instances
-provision_gce_startup_script_file: # "startup.sh"
-# Provide userdata to the gce instances
-provision_gce_user_data_file:
-
-rhsub_skip: true
 provision_custom_repositories: [
   {
     "name": "oso-rhui-rhel-server-releases",
@@ -124,10 +78,6 @@ provision_custom_repositories: [
   }
 ]
 
-##############
-# New settings
-##############
-
 openshift_gcp_prefix: "ci-{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}-"
 
 openshift_gcp_project: openshift-gce-devel-ci
@@ -137,34 +87,57 @@ openshift_gcp_zone: us-east1-c
 openshift_gcp_network_name: "{{ openshift_gcp_prefix }}network"
 
 openshift_gcp_iam_service_account: jenkins-ci-provisioner@openshift-gce-devel.iam.gserviceaccount.com
-openshift_gcp_iam_service_account_keyfile: "{{ inventory_dir }}/gce.json"
+openshift_gcp_iam_service_account_keyfile: "{{ files_dir }}/gce.json"
 
 openshift_gcp_ssh_private_key: /home/cloud-user/.ssh/google_compute_engine
 
 openshift_gcp_infra_network_instance_group: ig-m
 
-openshift_gcp_image: 'rhel-7'
-openshift_gcp_base_image: 'centos-7'
+# Instructs the launch job to build an AMI prior to cluster provisioning and then use that image
+openshift_gcp_build_image: true
+openshift_gcp_image: '{{ openshift_gcp_prefix }}images'
+openshift_gcp_base_image: 'rhel-7'
+openshift_gcp_root_image: 'centos-7'
 
-openshift_gcp_registry_bucket_keyfile: "{{ inventory_dir }}/gce.json"
+openshift_gcp_registry_bucket_keyfile: "{{ files_dir }}/gce.json"
 openshift_gcp_registry_bucket_name: "origin-ci-test-{{ lookup('env', 'INSTANCE_PREFIX') | mandatory }}"
 
+openshift_gcp_startup_script_file: "{{ files_dir }}/bootstrap-script.sh"
 openshift_gcp_node_group_config:
   - name: master
     suffix: m
-    tags: ocp-master
+    tags: ocp-master,ocp-infra-node,ocp-node
     machine_type: n1-standard-2
     boot_disk_size: 150
     scale: 1
+    bootstrap: true
+    wait_for_stable: true
   - name: node
     suffix: n
     tags: ocp-node
     machine_type: n1-standard-2
     boot_disk_size: 150
     scale: 3
+    bootstrap: true
   - name: node-flex
     suffix: nf
     tags: ocp-node
     machine_type: n1-standard-2
     boot_disk_size: 150
     scale: 0
+    bootstrap: true
+
+openshift_master_node_configs:
+- type: master
+  edits:
+  - key: kubeletArguments.node-labels
+    value: ['role=infra']
+- type: node
+  edits:
+  - key: kubeletArguments.node-labels
+    value: ['role=app']
+
+osm_controller_args: {"experimental-cluster-signing-duration": ["20m"]}
+openshift_master_bootstrap_enabled: true
+openshift_master_bootstrap_auto_approve: true
+openshift_master_bootstrap_auto_approver_node_selector: {"role": "infra"}