Merge pull request #2 from rajatchopra/master

Sync mode for running independently of PaaS to register nodes

Author: Rajat Chopra

README.md

@@ -31,7 +31,7 @@ Steps to create manually create an OpenShift cluster with openshift-sdn. This re
 
 On OpenShift master,
 
-	$ openshift start master  # start the master openshift server (also starts the etcd server by default)
+	$ openshift start master [--nodes=node1]  # start the master openshift server (also starts the etcd server by default) with an optional list of nodes
 	$ openshift-sdn           # assumes etcd is running at localhost:4001
 
 To add a node to the cluster, do the following on the node:
@@ -59,6 +59,8 @@ Back on the master, to finally register the node:
 
 Done. Repeat last two pieces to add more nodes. Create new pods from the master (or just docker containers on the minions), and see that the pods are indeed reachable from each other. 
 
+##### OpenShift? PaaS? Can I create a plain one just for Docker?
+
 #### Performance Note
 
 The current design has a long path for packets directed for the overlay network.

main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
+	"path"
 	"strings"
 	"syscall"
 	"time"
@@ -26,6 +27,7 @@ type CmdLineOpts struct {
 	master        bool
 	minion        bool
 	skipsetup     bool
+	sync          bool
 	help          bool
 }
 
@@ -44,6 +46,7 @@ func init() {
 	flag.BoolVar(&opts.master, "master", true, "Run in master mode")
 	flag.BoolVar(&opts.minion, "minion", false, "Run in minion mode")
 	flag.BoolVar(&opts.skipsetup, "skip-setup", false, "Skip the setup when in minion mode")
+	flag.BoolVar(&opts.sync, "sync", false, "Sync the minions directly to etcd-path (Do not wait for PaaS to do so!)")
 
 	flag.BoolVar(&opts.help, "help", false, "print this message")
 }
@@ -61,13 +64,19 @@ func newNetworkManager() controller.Controller {
 func newSubnetRegistry() registry.SubnetRegistry {
 	peers := strings.Split(opts.etcdEndpoints, ",")
 
+	subnetPath := path.Join(opts.etcdPath + "subnets")
+	minionPath := "/registry/minions/"
+	if opts.sync {
+		minionPath = path.Join(opts.etcdPath + "minions")
+	}
+
 	cfg := &registry.EtcdConfig{
 		Endpoints:  peers,
 		Keyfile:    opts.etcdKeyfile,
 		Certfile:   opts.etcdCertfile,
 		CAFile:     opts.etcdCAFile,
-		SubnetPath: opts.etcdPath,
-		MinionPath: "/registry/minions/",
+		SubnetPath: subnetPath,
+		MinionPath: minionPath,
 	}
 
 	for {
@@ -102,7 +111,7 @@ func main() {
 
 	be := newNetworkManager()
 	if opts.minion {
-		err := be.StartNode(opts.skipsetup)
+		err := be.StartNode(opts.sync, opts.skipsetup)
 		if err != nil {
 			return
 		}

ovs-simple/controller/controller.go

@@ -20,7 +20,7 @@ const (
 
 type Controller interface {
 	StartMaster() error
-	StartNode(skipsetup bool) error
+	StartNode(sync, skipsetup bool) error
 	AddNode(minionIP string) error
 	DeleteNode(minionIP string) error
 	Stop()
@@ -142,7 +142,18 @@ func (oc *OvsController) DeleteNode(minion string) error {
 	return oc.sm.DeleteSubnet(minion)
 }
 
-func (oc *OvsController) StartNode(skipsetup bool) error {
+func (oc *OvsController) syncWithMaster() error {
+	return oc.sm.CreateMinion(oc.hostName, oc.localIP)
+}
+
+func (oc *OvsController) StartNode(sync, skipsetup bool) error {
+	if sync {
+		err := oc.syncWithMaster()
+		if err != nil {
+			log.Errorf("Failed to register with master. (%s)", err.Error())
+			return err
+		}
+	}
 	err := oc.initSelfSubnet()
 	if err != nil {
 		log.Errorf("Failed to get subnet for this host. (%v)", err)
@@ -239,15 +250,15 @@ func (oc *OvsController) AddOFRules(minionip, subnet string) {
 	cookie := generateCookie(minionip)
 	if minionip == oc.localIP {
 		// self, so add the input rules
-		iprule := fmt.Sprintf("table=0,cookie=%s,priority=200,ip,in_port=10,nw_dst=%s,actions=output:9", cookie, subnet)
-		arprule := fmt.Sprintf("table=0,cookie=%s,priority=200,arp,in_port=10,nw_dst=%s,actions=output:9", cookie, subnet)
+		iprule := fmt.Sprintf("table=0,cookie=0x%s,priority=200,ip,in_port=10,nw_dst=%s,actions=output:9", cookie, subnet)
+		arprule := fmt.Sprintf("table=0,cookie=0x%s,priority=200,arp,in_port=10,nw_dst=%s,actions=output:9", cookie, subnet)
 		o, e := exec.Command("ovs-ofctl", "-O", "OpenFlow13", "add-flow", "br0", iprule).CombinedOutput()
 		log.Infof("Output of adding %s: %s (%v)", iprule, o, e)
 		o, e = exec.Command("ovs-ofctl", "-O", "OpenFlow13", "add-flow", "br0", arprule).CombinedOutput()
 		log.Infof("Output of adding %s: %s (%v)", arprule, o, e)
 	} else {
-		iprule := fmt.Sprintf("table=0,cookie=%s,priority=200,ip,in_port=9,nw_dst=%s,actions=set_field:%s->tun_dst,output:10", cookie, subnet, minionip)
-		arprule := fmt.Sprintf("table=0,cookie=%s,priority=200,arp,in_port=9,nw_dst=%s,actions=set_field:%s->tun_dst,output:10", cookie, subnet, minionip)
+		iprule := fmt.Sprintf("table=0,cookie=0x%s,priority=200,ip,in_port=9,nw_dst=%s,actions=set_field:%s->tun_dst,output:10", cookie, subnet, minionip)
+		arprule := fmt.Sprintf("table=0,cookie=0x%s,priority=200,arp,in_port=9,nw_dst=%s,actions=set_field:%s->tun_dst,output:10", cookie, subnet, minionip)
 		o, e := exec.Command("ovs-ofctl", "-O", "OpenFlow13", "add-flow", "br0", iprule).CombinedOutput()
 		log.Infof("Output of adding %s: %s (%v)", iprule, o, e)
 		o, e = exec.Command("ovs-ofctl", "-O", "OpenFlow13", "add-flow", "br0", arprule).CombinedOutput()
@@ -259,15 +270,15 @@ func (oc *OvsController) DelOFRules(minion string) {
 	log.Infof("Calling del rules for %s", minion)
 	cookie := generateCookie(minion)
 	if minion == oc.localIP {
-		iprule := fmt.Sprintf("table=0,cookie=%s,ip,in_port=10", cookie)
-		arprule := fmt.Sprintf("table=0,cookie=%s,arp,in_port=10", cookie)
+		iprule := fmt.Sprintf("table=0,cookie=0x%s/0xffffffff,ip,in_port=10", cookie)
+		arprule := fmt.Sprintf("table=0,cookie=0x%s/0xffffffff,arp,in_port=10", cookie)
 		o, e := exec.Command("ovs-ofctl", "-O", "OpenFlow13", "del-flows", "br0", iprule).CombinedOutput()
 		log.Infof("Output of deleting local ip rules %s (%v)", o, e)
 		o, e = exec.Command("ovs-ofctl", "-O", "OpenFlow13", "del-flows", "br0", arprule).CombinedOutput()
 		log.Infof("Output of deleting local ip rules %s (%v)", o, e)
 	} else {
-		iprule := fmt.Sprintf("table=0,cookie=%s/0xffffffff,ip,in_port=9", cookie)
-		arprule := fmt.Sprintf("table=0,cookie=%s/0xffffffff,arp,in_port=9", cookie)
+		iprule := fmt.Sprintf("table=0,cookie=0x%s/0xffffffff,ip,in_port=9", cookie)
+		arprule := fmt.Sprintf("table=0,cookie=0x%s/0xffffffff,arp,in_port=9", cookie)
 		o, e := exec.Command("ovs-ofctl", "-O", "OpenFlow13", "del-flows", "br0", iprule).CombinedOutput()
 		log.Infof("Output of deleting %s: %s (%v)", iprule, o, e)
 		o, e = exec.Command("ovs-ofctl", "-O", "OpenFlow13", "del-flows", "br0", arprule).CombinedOutput()

pkg/registry/registry.go

@@ -25,6 +25,7 @@ type SubnetRegistry interface {
 	GetSubnet(minion string) (*Subnet, error)
 	DeleteSubnet(minion string) error
 	CreateSubnet(sn string, sub *Subnet) (*etcd.Response, error)
+	CreateMinion(minion string, data string) error
 	WatchSubnets(rev uint64, receiver chan *SubnetEvent, stop chan bool) error
 	GetMinions() (*[]string, error)
 	WatchMinions(rev uint64, receiver chan *MinionEvent, stop chan bool) error
@@ -200,6 +201,21 @@ func (sub *EtcdSubnetRegistry) DeleteSubnet(minion string) error {
 	return err
 }
 
+func (sub *EtcdSubnetRegistry) CreateMinion(minion string, data string) error {
+	key := path.Join(sub.etcdCfg.MinionPath, minion)
+	_, err := sub.client().Get(key, false, false)
+	if err != nil {
+		// good, it does not exist, write it
+		_, err = sub.client().Create(key, data, 0)
+		if err != nil {
+			log.Errorf("Failed to write new subnet to etcd - %v", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (sub *EtcdSubnetRegistry) CreateSubnet(minion string, subnet *Subnet) (*etcd.Response, error) {
 	subbytes, _ := json.Marshal(subnet)
 	data := string(subbytes)