Skip to content

Commit eb016b3

Browse files
openshift-merge-robotopenshift-merge-robot
authored
Merge pull request #18956 from deads2k/up-21-componentinstall
Automatic merge from submit-queue. make registry installation a component Demonstrate a simple way of making a component and installing it. This builds on previous pulls and shows how we can start define an interface. I think after we switch pieces over, we'll find points of commonality as their entry points. I suspect they will include: 1. cluster-admin.kubeconfig 2. docker helper? We should try to switch this to actually installing with pods. 3. uninstall API. Something to remove it anyway. 4. idempotent. We will call it *on every cluster up* /assign @mfojtik /assign @soltysh
2 parents 9f45ccb + be6dc5e commit eb016b3

File tree

4 files changed

+160
-99
lines changed

4 files changed

+160
-99
lines changed

‎pkg/oc/bootstrap/clusterup/components/registry/registry_install.go‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package registry
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path"
7+
8+
"github.com/golang/glog"
9+
10+
apierrors "k8s.io/apimachinery/pkg/api/errors"
11+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/client-go/kubernetes"
13+
"k8s.io/client-go/rest"
14+
15+
"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/componentinstall"
16+
"github.com/openshift/origin/pkg/oc/bootstrap/docker/dockerhelper"
17+
"github.com/openshift/origin/pkg/oc/bootstrap/docker/openshift"
18+
"github.com/openshift/origin/pkg/oc/bootstrap/docker/run"
19+
"github.com/openshift/origin/pkg/oc/errors"
20+
securityclient "github.com/openshift/origin/pkg/security/generated/internalclientset/typed/security/internalversion"
21+
)
22+
23+
const (
24+
DefaultNamespace = "default"
25+
SvcDockerRegistry = "docker-registry"
26+
masterConfigDir = "/var/lib/origin/openshift.local.config/master"
27+
// This is needed because of NO_PROXY cannot handle the CIDR range
28+
RegistryServiceClusterIP = "172.30.1.1"
29+
)
30+
31+
type RegistryComponentOptions struct {
32+
ClusterAdminKubeConfig *rest.Config
33+
34+
OCImage string
35+
MasterConfigDir string
36+
Images string
37+
PVDir string
38+
}
39+
40+
func (r *RegistryComponentOptions) Name() string {
41+
return "openshift-image-registry"
42+
}
43+
44+
func (r *RegistryComponentOptions) Install(dockerClient dockerhelper.Interface, logdir string) error {
45+
kubeClient, err := kubernetes.NewForConfig(r.ClusterAdminKubeConfig)
46+
_, err = kubeClient.Core().Services(DefaultNamespace).Get(SvcDockerRegistry, metav1.GetOptions{})
47+
if err == nil {
48+
// If there's no error, the registry already exists
49+
return nil
50+
}
51+
if !apierrors.IsNotFound(err) {
52+
return errors.NewError("error retrieving docker registry service").WithCause(err)
53+
}
54+
55+
imageRunHelper := run.NewRunHelper(dockerhelper.NewHelper(dockerClient)).New()
56+
glog.Infof("Running %q", r.Name())
57+
58+
securityClient, err := securityclient.NewForConfig(r.ClusterAdminKubeConfig)
59+
if err != nil {
60+
return err
61+
}
62+
err = openshift.AddSCCToServiceAccount(securityClient, "privileged", "registry", "default", os.Stdout)
63+
if err != nil {
64+
return errors.NewError("cannot add privileged SCC to registry service account").WithCause(err)
65+
}
66+
67+
// Obtain registry markup. The reason it is not created outright is because
68+
// we need to modify the ClusterIP of the registry service. The command doesn't
69+
// have an option to set it.
70+
flags := []string{
71+
"adm",
72+
"registry",
73+
"--loglevel=8",
74+
// We need to set the ClusterIP for registry in order to be able to set the NO_PROXY no predicable
75+
// IP address as NO_PROXY does not support CIDR format.
76+
// TODO: We should switch the cluster up registry to use DNS.
77+
"--cluster-ip=" + RegistryServiceClusterIP,
78+
"--config=" + masterConfigDir + "/admin.kubeconfig",
79+
fmt.Sprintf("--images=%s", r.Images),
80+
fmt.Sprintf("--mount-host=%s", path.Join(r.PVDir, "registry")),
81+
}
82+
_, stdout, stderr, rc, err := imageRunHelper.Image(r.OCImage).
83+
Privileged().
84+
DiscardContainer().
85+
HostNetwork().
86+
HostPid().
87+
Bind(r.MasterConfigDir + ":" + masterConfigDir).
88+
Entrypoint("oc").
89+
Command(flags...).Output()
90+
91+
if err := componentinstall.LogContainer(logdir, r.Name(), stdout, stderr); err != nil {
92+
glog.Errorf("error logging %q: %v", r.Name(), err)
93+
}
94+
if err != nil {
95+
return errors.NewError("could not run %q: %v", r.Name(), err).WithCause(err)
96+
}
97+
if rc != 0 {
98+
return errors.NewError("could not run %q: rc==%v", r.Name(), rc)
99+
}
100+
101+
return nil
102+
}

‎pkg/oc/bootstrap/docker/openshift/admin.go‎

Lines changed: 1 addition & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"fmt"
66
"io"
77
"os"
8-
"path"
98
"path/filepath"
109

1110
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -27,79 +26,14 @@ import (
2726
)
2827

2928
const (
30-
DefaultNamespace = "default"
31-
RegistryServiceName = "docker-registry"
32-
RegistryServiceAccountName = "registry"
33-
// This is needed because of NO_PROXY cannot handle the CIDR range
34-
RegistryServiceClusterIP = "172.30.1.1"
29+
DefaultNamespace = "default"
3530
RouterServiceAccountName = "router"
3631
RouterServiceName = "router"
3732

3833
masterConfigDir = "/var/lib/origin/openshift.local.config/master"
3934
routerCertPath = masterConfigDir + "/router.pem"
4035
)
4136

42-
// InstallRegistry checks whether a registry is installed and installs one if not already installed
43-
func (h *Helper) InstallRegistry(dockerClient dockerhelper.Interface, ocImage string, kubeClient kclientset.Interface, f *clientcmd.Factory, configDir, logdir, images, pvDir string, out, errout io.Writer) error {
44-
_, err := kubeClient.Core().Services(DefaultNamespace).Get(RegistryServiceName, metav1.GetOptions{})
45-
if err == nil {
46-
glog.V(3).Infof("The %q service is already present, skipping installation", RegistryServiceName)
47-
// If there's no error, the registry already exists
48-
return nil
49-
}
50-
if !apierrors.IsNotFound(err) {
51-
return errors.NewError("error retrieving docker-registry service").WithCause(err).WithDetails(h.OriginLog())
52-
}
53-
54-
componentName := "install-registry"
55-
imageRunHelper := run.NewRunHelper(dockerhelper.NewHelper(dockerClient)).New()
56-
glog.Infof("Running %q", componentName)
57-
58-
securityClient, err := f.OpenshiftInternalSecurityClient()
59-
if err != nil {
60-
return err
61-
}
62-
err = AddSCCToServiceAccount(securityClient.Security(), "privileged", RegistryServiceAccountName, "default", out)
63-
if err != nil {
64-
return errors.NewError("cannot add privileged SCC to registry service account").WithCause(err).WithDetails(h.OriginLog())
65-
}
66-
67-
masterDir := filepath.Join(configDir, "master")
68-
69-
// Obtain registry markup. The reason it is not created outright is because
70-
// we need to modify the ClusterIP of the registry service. The command doesn't
71-
// have an option to set it.
72-
flags := []string{
73-
"adm",
74-
"registry",
75-
"--loglevel=8",
76-
// We need to set the ClusterIP for registry in order to be able to set the NO_PROXY no predicable
77-
// IP address as NO_PROXY does not support CIDR format.
78-
// TODO: We should switch the cluster up registry to use DNS.
79-
"--cluster-ip=" + RegistryServiceClusterIP,
80-
"--config=" + masterConfigDir + "/admin.kubeconfig",
81-
fmt.Sprintf("--images=%s", images),
82-
fmt.Sprintf("--mount-host=%s", path.Join(pvDir, "registry")),
83-
}
84-
_, stdout, stderr, rc, err := imageRunHelper.Image(ocImage).
85-
DiscardContainer().
86-
HostNetwork().
87-
Bind(masterDir + ":" + masterConfigDir).
88-
Entrypoint("oc").
89-
Command(flags...).Output()
90-
91-
if err := componentinstall.LogContainer(logdir, componentName, stdout, stderr); err != nil {
92-
glog.Errorf("error logging %q: %v", componentName, err)
93-
}
94-
if err != nil {
95-
return errors.NewError("could not run %q: %v", componentName, err).WithCause(err)
96-
}
97-
if rc != 0 {
98-
return errors.NewError("could not run %q: rc==%v", componentName, rc)
99-
}
100-
return err
101-
}
102-
10337
// InstallRouter installs a default router on the OpenShift server
10438
func (h *Helper) InstallRouter(dockerClient dockerhelper.Interface, ocImage string, kubeClient kclientset.Interface, f *clientcmd.Factory, configDir, logdir, images, hostIP string, portForwarding bool, out, errout io.Writer) error {
10539
_, err := kubeClient.Core().Services(DefaultNamespace).Get(RouterServiceName, metav1.GetOptions{})

‎pkg/oc/bootstrap/docker/up.go‎

Lines changed: 37 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ import (
1717
cliconfig "github.com/docker/docker/cli/config"
1818
dockerclient "github.com/docker/docker/client"
1919
"github.com/golang/glog"
20-
"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/tmpformac"
2120
"github.com/spf13/cobra"
2221
"github.com/spf13/pflag"
2322
"golang.org/x/net/context"
@@ -37,6 +36,8 @@ import (
3736
"github.com/openshift/origin/pkg/cmd/util/variable"
3837
"github.com/openshift/origin/pkg/oc/bootstrap"
3938
"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/componentinstall"
39+
"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/components/registry"
40+
"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/tmpformac"
4041
"github.com/openshift/origin/pkg/oc/bootstrap/docker/dockerhelper"
4142
"github.com/openshift/origin/pkg/oc/bootstrap/docker/dockermachine"
4243
"github.com/openshift/origin/pkg/oc/bootstrap/docker/errors"
@@ -484,12 +485,35 @@ func (c *ClusterUpConfig) Start(out io.Writer) error {
484485
}
485486
taskPrinter.Success()
486487

487-
// Install a registry
488-
taskPrinter.StartTask("Installing registry")
489-
if err := c.InstallRegistry(out); err != nil {
490-
return taskPrinter.ToError(err)
488+
clusterAdminKubeConfigBytes, err := ioutil.ReadFile(path.Join(c.LocalConfigDir, "master", "admin.kubeconfig"))
489+
if err != nil {
490+
return err
491+
}
492+
clusterAdminKubeConfig, err := kclientcmd.RESTConfigFromKubeConfig(clusterAdminKubeConfigBytes)
493+
if err != nil {
494+
return err
495+
}
496+
497+
// TODO, now we build up a set of things to install here. We build the list so that we can install everything in
498+
// TODO parallel to avoid anyone accidentally introducing dependencies. We'll start with migrating what we have
499+
// TODO and then we'll try to clean it up.
500+
registryInstall := &registry.RegistryComponentOptions{
501+
ClusterAdminKubeConfig: clusterAdminKubeConfig,
502+
503+
OCImage: c.openshiftImage(),
504+
MasterConfigDir: path.Join(c.LocalConfigDir, "master"),
505+
Images: c.imageFormat(),
506+
PVDir: c.HostPersistentVolumesDir,
507+
}
508+
509+
componentsToInstall := []componentinstall.Component{}
510+
componentsToInstall = append(componentsToInstall, c.ImportInitialObjectsComponents(c.Out)...)
511+
componentsToInstall = append(componentsToInstall, registryInstall)
512+
513+
err = componentinstall.InstallComponents(componentsToInstall, c.GetDockerClient(), path.Join(c.BaseTempDir, "logs"))
514+
if err != nil {
515+
return err
491516
}
492-
taskPrinter.Success()
493517

494518
// Install a router
495519
taskPrinter.StartTask("Installing router")
@@ -507,13 +531,6 @@ func (c *ClusterUpConfig) Start(out io.Writer) error {
507531
taskPrinter.Success()
508532
}
509533

510-
// Import default image streams
511-
taskPrinter.StartTask("Importing default data router")
512-
if err := c.ImportInitialObjects(out); err != nil {
513-
return taskPrinter.ToError(err)
514-
}
515-
taskPrinter.Success()
516-
517534
// Install logging
518535
if c.ShouldInstallLogging {
519536
taskPrinter.StartTask("Installing logging")
@@ -835,7 +852,7 @@ func (c *ClusterUpConfig) determineServerIP(out io.Writer) (string, []string, er
835852

836853
// updateNoProxy will add some default values to the NO_PROXY setting if they are not present
837854
func (c *ClusterUpConfig) updateNoProxy() {
838-
values := []string{"127.0.0.1", c.ServerIP, "localhost", openshift.ServiceCatalogServiceIP, openshift.RegistryServiceClusterIP}
855+
values := []string{"127.0.0.1", c.ServerIP, "localhost", openshift.ServiceCatalogServiceIP, registry.RegistryServiceClusterIP}
839856
ipFromServer, err := c.OpenShiftHelper().ServerIP()
840857
if err == nil {
841858
values = append(values, ipFromServer)
@@ -885,19 +902,6 @@ func (c *ClusterUpConfig) imageFormat() string {
885902
return fmt.Sprintf("%s-${component}:%s", c.Image, c.ImageVersion)
886903
}
887904

888-
// InstallRegistry installs the OpenShift registry on the server
889-
func (c *ClusterUpConfig) InstallRegistry(out io.Writer) error {
890-
_, kubeClient, err := c.Clients()
891-
if err != nil {
892-
return err
893-
}
894-
f, err := c.Factory()
895-
if err != nil {
896-
return err
897-
}
898-
return c.OpenShiftHelper().InstallRegistry(c.GetDockerClient(), c.openshiftImage(), kubeClient, f, c.LocalConfigDir, path.Join(c.BaseTempDir, "logs"), c.imageFormat(), c.HostPersistentVolumesDir, out, os.Stderr)
899-
}
900-
901905
// InstallRouter installs a default router on the server
902906
func (c *ClusterUpConfig) InstallRouter(out io.Writer) error {
903907
_, kubeClient, err := c.Clients()
@@ -938,7 +942,8 @@ func (c *ClusterUpConfig) InstallWebConsole(out io.Writer) error {
938942
return c.OpenShiftHelper().InstallWebConsole(f, c.imageFormat(), c.ServerLogLevel, publicURL, masterURL, loggingURL, metricsURL)
939943
}
940944

941-
func (c *ClusterUpConfig) ImportInitialObjects(out io.Writer) error {
945+
// TODO this should become a separate thing we can install, like registry
946+
func (c *ClusterUpConfig) ImportInitialObjectsComponents(out io.Writer) []componentinstall.Component {
942947
componentsToInstall := []componentinstall.Component{}
943948
componentsToInstall = append(componentsToInstall,
944949
c.makeObjectImportInstallationComponentsOrDie(out, openshift.Namespace, map[string]string{
@@ -953,7 +958,7 @@ func (c *ClusterUpConfig) ImportInitialObjects(out io.Writer) error {
953958
componentsToInstall = append(componentsToInstall,
954959
c.makeObjectImportInstallationComponentsOrDie(out, openshift.InfraNamespace, internalCurrentTemplateLocations)...)
955960

956-
return componentinstall.InstallComponents(componentsToInstall, c.GetDockerClient(), path.Join(c.BaseTempDir, "logs"))
961+
return componentsToInstall
957962
}
958963

959964
// InstallLogging will start the installation of logging components
@@ -1120,9 +1125,9 @@ func (c *ClusterUpConfig) checkProxySettings() string {
11201125
if len(dockerHTTPProxy) > 0 || len(dockerHTTPSProxy) > 0 {
11211126
dockerNoProxyList := strings.Split(dockerNoProxy, ",")
11221127
dockerNoProxySet := sets.NewString(dockerNoProxyList...)
1123-
if !dockerNoProxySet.Has(openshift.RegistryServiceClusterIP) {
1128+
if !dockerNoProxySet.Has(registry.RegistryServiceClusterIP) {
11241129
warnings = append(warnings, fmt.Sprintf("A proxy is configured for Docker, however %[1]s is not included in its NO_PROXY list.\n"+
1125-
" %[1]s needs to be included in the Docker daemon's NO_PROXY environment variable so pushes to the local OpenShift registry can succeed.", openshift.RegistryServiceClusterIP))
1130+
" %[1]s needs to be included in the Docker daemon's NO_PROXY environment variable so pushes to the local OpenShift registry can succeed.", registry.RegistryServiceClusterIP))
11261131
}
11271132
}
11281133

@@ -1361,7 +1366,7 @@ func (c *ClusterUpConfig) ShouldInitializeData() bool {
13611366
return true
13621367
}
13631368

1364-
if _, err = kclient.Core().Services(openshift.DefaultNamespace).Get(openshift.RegistryServiceName, metav1.GetOptions{}); err != nil {
1369+
if _, err = kclient.Core().Services(openshift.DefaultNamespace).Get(registry.SvcDockerRegistry, metav1.GetOptions{}); err != nil {
13651370
return true
13661371
}
13671372

‎vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/tools/clientcmd/client_config.go‎

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)