func (t *tcShaper) execAndLog(cmdStr string, args ...string) error {
	glog.V(6).Infof("Running: %s %s", cmdStr, strings.Join(args, " "))
	cmd := t.e.Command(cmdStr, args...)
	out, err := cmd.CombinedOutput()
	glog.V(6).Infof("Output from tc: %s", string(out))
	return err
}

func chooseHostInterfaceFromRoute(inFile io.Reader, nw networkInterfacer) (net.IP, error) {
	routes, err := getRoutes(inFile)
	if err != nil {
		return nil, err
	}
	zero := net.IP{0, 0, 0, 0}
	var finalIP net.IP
	for i := range routes {
		//find interface with gateway
		if routes[i].Destination.Equal(zero) {
			glog.V(4).Infof("Default route transits interface %q", routes[i].Interface)
			finalIP, err := getIPFromInterface(routes[i].Interface, nw)
			if err != nil {
				return nil, err
			}
			if finalIP != nil {
				glog.V(4).Infof("Choosing IP %v ", finalIP)
				return finalIP, nil
			}
		}
	}
	glog.V(4).Infof("No valid IP found")
	if finalIP == nil {
		return nil, fmt.Errorf("Unable to select an IP.")
	}
	return nil, nil
}

//getFinalIP method receives all the IP addrs of a Interface
//and returns a nil if the address is Loopback, Ipv6, link-local or nil.
//It returns a valid IPv4 if an Ipv4 address is found in the array.
func getFinalIP(addrs []net.Addr) (net.IP, error) {
	if len(addrs) > 0 {
		for i := range addrs {
			glog.V(4).Infof("Checking addr  %s.", addrs[i].String())
			ip, _, err := net.ParseCIDR(addrs[i].String())
			if err != nil {
				return nil, err
			}
			//Only IPv4
			//TODO : add IPv6 support
			if ip.To4() != nil {
				if !ip.IsLoopback() && !ip.IsLinkLocalMulticast() && !ip.IsLinkLocalUnicast() {
					glog.V(4).Infof("IP found %v", ip)
					return ip, nil
				} else {
					glog.V(4).Infof("Loopback/link-local found %v", ip)
				}
			} else {
				glog.V(4).Infof("%v is not a valid IPv4 address", ip)
			}

		}
	}
	return nil, nil
}

// TODO: should take a writer, not []byte
func (writer *NsenterWriter) WriteFile(filename string, data []byte, perm os.FileMode) error {
	cmd := "nsenter"
	base_args := []string{
		"--mount=/rootfs/proc/1/ns/mnt",
		"--",
	}

	echo_args := append(base_args, "sh", "-c", fmt.Sprintf("cat > %s", filename))
	glog.V(5).Infof("Command to write data to file: %v %v", cmd, echo_args)
	command := exec.Command(cmd, echo_args...)
	command.Stdin = bytes.NewBuffer(data)
	outputBytes, err := command.CombinedOutput()
	if err != nil {
		glog.Errorf("Output from writing to %q: %v", filename, string(outputBytes))
		return err
	}

	chmod_args := append(base_args, "chmod", fmt.Sprintf("%o", perm), filename)
	glog.V(5).Infof("Command to change permissions to file: %v %v", cmd, chmod_args)
	outputBytes, err = exec.Command(cmd, chmod_args...).CombinedOutput()
	if err != nil {
		glog.Errorf("Output from chmod command: %v", string(outputBytes))
		return err
	}

	return nil
}

// IsLikelyNotMountPoint determines whether a path is a mountpoint by calling findmnt
// in the host's root mount namespace.
func (n *NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) {
	file, err := filepath.Abs(file)
	if err != nil {
		return true, err
	}

	args := []string{"--mount=/rootfs/proc/1/ns/mnt", "--", n.absHostPath("findmnt"), "-o", "target", "--noheadings", "--target", file}
	glog.V(5).Infof("findmnt command: %v %v", nsenterPath, args)

	exec := exec.New()
	out, err := exec.Command(nsenterPath, args...).CombinedOutput()
	if err != nil {
		glog.Errorf("Failed to nsenter mount, return file doesn't exist: %v", err)
		// If the command itself is correct, then if we encountered error
		// then most likely this means that the directory does not exist.
		return true, os.ErrNotExist
	}
	strOut := strings.TrimSuffix(string(out), "\n")

	glog.V(5).Infof("IsLikelyNotMountPoint findmnt output: %v", strOut)
	if strOut == file {
		return false, nil
	}

	return true, nil
}

// Body makes the request use obj as the body. Optional.
// If obj is a string, try to read a file of that name.
// If obj is a []byte, send it directly.
// If obj is an io.Reader, use it directly.
// If obj is a runtime.Object, marshal it correctly, and set Content-Type header.
// Otherwise, set an error.
func (r *Request) Body(obj interface{}) *Request {
	if r.err != nil {
		return r
	}
	switch t := obj.(type) {
	case string:
		data, err := ioutil.ReadFile(t)
		if err != nil {
			r.err = err
			return r
		}
		glog.V(8).Infof("Request Body: %s", string(data))
		r.body = bytes.NewBuffer(data)
	case []byte:
		glog.V(8).Infof("Request Body: %s", string(t))
		r.body = bytes.NewBuffer(t)
	case io.Reader:
		r.body = t
	case runtime.Object:
		data, err := r.codec.Encode(t)
		if err != nil {
			r.err = err
			return r
		}
		glog.V(8).Infof("Request Body: %s", string(data))
		r.body = bytes.NewBuffer(data)
		r.SetHeader("Content-Type", "application/json")
	default:
		r.err = fmt.Errorf("unknown type used for body: %+v", obj)
	}
	return r
}

// receive reads result from the decoder in a loop and sends down the result channel.
func (sw *StreamWatcher) receive() {
	defer close(sw.result)
	defer sw.Stop()
	defer util.HandleCrash()
	for {
		action, obj, err := sw.source.Decode()
		if err != nil {
			// Ignore expected error.
			if sw.stopping() {
				return
			}
			switch err {
			case io.EOF:
				// watch closed normally
			case io.ErrUnexpectedEOF:
				glog.V(1).Infof("Unexpected EOF during watch stream event decoding: %v", err)
			default:
				msg := "Unable to decode an event from the watch stream: %v"
				if util.IsProbableEOF(err) {
					glog.V(5).Infof(msg, err)
				} else {
					glog.Errorf(msg, err)
				}
			}
			return
		}
		sw.result <- Event{
			Type:   action,
			Object: obj,
		}
	}
}

// doNsenterMount nsenters the host's mount namespace and performs the
// requested mount.
func (n *NsenterMounter) doNsenterMount(source, target, fstype string, options []string) error {
	glog.V(5).Infof("nsenter Mounting %s %s %s %v", source, target, fstype, options)
	args := n.makeNsenterArgs(source, target, fstype, options)

	glog.V(5).Infof("Mount command: %v %v", nsenterPath, args)
	exec := exec.New()
	outputBytes, err := exec.Command(nsenterPath, args...).CombinedOutput()
	if len(outputBytes) != 0 {
		glog.V(5).Infof("Output from mount command: %v", string(outputBytes))
	}

	return err
}

// Decode unmarshals the next object from the underlying stream into the
// provide object, or returns an error.
func (d *YAMLOrJSONDecoder) Decode(into interface{}) error {
	if d.decoder == nil {
		buffer, isJSON := guessJSONStream(d.r, d.bufferSize)
		if isJSON {
			glog.V(4).Infof("decoding stream as JSON")
			d.decoder = json.NewDecoder(buffer)
		} else {
			glog.V(4).Infof("decoding stream as YAML")
			d.decoder = NewYAMLToJSONDecoder(buffer)
		}
	}
	return d.decoder.Decode(into)
}

// DebugWrappers wraps a round tripper and logs based on the current log level.
func DebugWrappers(rt http.RoundTripper) http.RoundTripper {
	switch {
	case bool(glog.V(9)):
		rt = newDebuggingRoundTripper(rt, debugCurlCommand, debugURLTiming, debugResponseHeaders)
	case bool(glog.V(8)):
		rt = newDebuggingRoundTripper(rt, debugJustURL, debugRequestHeaders, debugResponseStatus, debugResponseHeaders)
	case bool(glog.V(7)):
		rt = newDebuggingRoundTripper(rt, debugJustURL, debugRequestHeaders, debugResponseStatus)
	case bool(glog.V(6)):
		rt = newDebuggingRoundTripper(rt, debugURLTiming)
	}

	return rt
}

// Releases the lock associated with the specified ID.
// Returns an error if the specified ID doesn't exist.
func (km *keyMutex) UnlockKey(id string) error {
	glog.V(5).Infof("UnlockKey(...) called for id %q\r\n", id)
	km.RLock()
	defer km.RUnlock()
	mutex, exists := km.mutexMap[id]
	if !exists {
		return fmt.Errorf("id %q not found", id)
	}
	glog.V(5).Infof("UnlockKey(...) for id. Mutex found, trying to unlock it. %q\r\n", id)

	mutex.Unlock()
	glog.V(5).Infof("UnlockKey(...) for id %q completed.\r\n", id)
	return nil
}

// handle implements a websocket handler.
func (conn *Conn) handle(ws *websocket.Conn) {
	defer conn.Close()
	conn.initialize(ws)

	for {
		conn.resetTimeout()
		var data []byte
		if err := websocket.Message.Receive(ws, &data); err != nil {
			if err != io.EOF {
				glog.Errorf("Error on socket receive: %v", err)
			}
			break
		}
		if len(data) == 0 {
			continue
		}
		channel := data[0]
		if conn.codec == base64Codec {
			channel = channel - '0'
		}
		data = data[1:]
		if int(channel) >= len(conn.channels) {
			glog.V(6).Infof("Frame is targeted for a reader %d that is not valid, possible protocol error", channel)
			continue
		}
		if _, err := conn.channels[channel].DataFromSocket(data); err != nil {
			glog.Errorf("Unable to write frame to %d: %v\n%s", channel, err, string(data))
			continue
		}
	}
}

func init() {
	groupMeta, err := latest.RegisterGroup("componentconfig")
	if err != nil {
		glog.V(4).Infof("%v", err)
		return
	}

	registeredGroupVersions := registered.GroupVersionsForGroup("componentconfig")
	groupVersion := registeredGroupVersions[0]
	*groupMeta = latest.GroupMeta{
		GroupVersion: groupVersion,
		Codec:        runtime.CodecFor(api.Scheme, groupVersion.String()),
	}

	var versions []string
	worstToBestGroupVersions := []unversioned.GroupVersion{}
	for i := len(registeredGroupVersions) - 1; i >= 0; i-- {
		versions = append(versions, registeredGroupVersions[i].Version)
		worstToBestGroupVersions = append(worstToBestGroupVersions, registeredGroupVersions[i])
	}
	groupMeta.Versions = versions
	groupMeta.GroupVersions = registeredGroupVersions

	groupMeta.SelfLinker = runtime.SelfLinker(accessor)

	// the list of kinds that are scoped at the root of the api hierarchy
	// if a kind is not enumerated here, it is assumed to have a namespace scope
	rootScoped := sets.NewString()

	ignoredKinds := sets.NewString()

	groupMeta.RESTMapper = api.NewDefaultRESTMapper(worstToBestGroupVersions, interfacesFor, importPrefix, ignoredKinds, rootScoped)
	api.RegisterRESTMapper(groupMeta.RESTMapper)
	groupMeta.InterfacesFor = interfacesFor
}

// Stream opens a protocol streamer to the server and streams until a client closes
// the connection or the server disconnects.
func (e *streamExecutor) Stream(stdin io.Reader, stdout, stderr io.Writer, tty bool) error {
	supportedProtocols := []string{StreamProtocolV2Name, StreamProtocolV1Name}
	conn, protocol, err := e.Dial(supportedProtocols...)
	if err != nil {
		return err
	}
	defer conn.Close()

	var streamer streamProtocolHandler

	switch protocol {
	case StreamProtocolV2Name:
		streamer = &streamProtocolV2{
			stdin:  stdin,
			stdout: stdout,
			stderr: stderr,
			tty:    tty,
		}
	case "":
		glog.V(4).Infof("The server did not negotiate a streaming protocol version. Falling back to %s", StreamProtocolV1Name)
		fallthrough
	case StreamProtocolV1Name:
		streamer = &streamProtocolV1{
			stdin:  stdin,
			stdout: stdout,
			stderr: stderr,
			tty:    tty,
		}
	}

	return streamer.stream(conn)
}

// runs all reload funcs to re-sync iptables rules
func (runner *runner) reload() {
	glog.V(1).Infof("reloading iptables rules")

	for _, f := range runner.reloadFuncs {
		f()
	}
}

// Unmount runs umount(8) in the host's mount namespace.
func (n *NsenterMounter) Unmount(target string) error {
	args := []string{
		"--mount=/rootfs/proc/1/ns/mnt",
		"--",
		n.absHostPath("umount"),
		target,
	}

	glog.V(5).Infof("Unmount command: %v %v", nsenterPath, args)
	exec := exec.New()
	outputBytes, err := exec.Command(nsenterPath, args...).CombinedOutput()
	if len(outputBytes) != 0 {
		glog.V(5).Infof("Output from mount command: %v", string(outputBytes))
	}

	return err
}

// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
	glog.V(5).Infof("Unmounting %s", target)
	command := exec.Command("umount", target)
	output, err := command.CombinedOutput()
	if err != nil {
		return fmt.Errorf("Unmount failed: %v\nUnmounting arguments: %s\nOutput: %s\n", err, target, string(output))
	}
	return nil
}

func (runner *runner) run(op operation, args []string) ([]byte, error) {
	iptablesCmd := runner.iptablesCommand()

	fullArgs := append(runner.waitFlag, string(op))
	fullArgs = append(fullArgs, args...)
	glog.V(4).Infof("running iptables %s %v", string(op), args)
	return runner.exec.Command(iptablesCmd, fullArgs...).CombinedOutput()
	// Don't log err here - callers might not think it is an error.
}

// Executes the rule check without using the "-C" flag, instead parsing iptables-save.
// Present for compatibility with <1.4.11 versions of iptables.  This is full
// of hack and half-measures.  We should nix this ASAP.
func (runner *runner) checkRuleWithoutCheck(table Table, chain Chain, args ...string) (bool, error) {
	glog.V(1).Infof("running iptables-save -t %s", string(table))
	out, err := runner.exec.Command(cmdIptablesSave, "-t", string(table)).CombinedOutput()
	if err != nil {
		return false, fmt.Errorf("error checking rule: %v", err)
	}

	// Sadly, iptables has inconsistent quoting rules for comments. Just remove all quotes.
	// Also, quoted multi-word comments (which are counted as a single arg)
	// will be unpacked into multiple args,
	// in order to compare against iptables-save output (which will be split at whitespace boundary)
	// e.g. a single arg('"this must be before the NodePort rules"') will be unquoted and unpacked into 7 args.
	var argsCopy []string
	for i := range args {
		tmpField := strings.Trim(args[i], "\"")
		argsCopy = append(argsCopy, strings.Fields(tmpField)...)
	}
	argset := sets.NewString(argsCopy...)

	for _, line := range strings.Split(string(out), "\n") {
		var fields = strings.Fields(line)

		// Check that this is a rule for the correct chain, and that it has
		// the correct number of argument (+2 for "-A <chain name>")
		if !strings.HasPrefix(line, fmt.Sprintf("-A %s", string(chain))) || len(fields) != len(argsCopy)+2 {
			continue
		}

		// Sadly, iptables has inconsistent quoting rules for comments.
		// Just remove all quotes.
		for i := range fields {
			fields[i] = strings.Trim(fields[i], "\"")
		}

		// TODO: This misses reorderings e.g. "-x foo ! -y bar" will match "! -x foo -y bar"
		if sets.NewString(fields...).IsSuperset(argset) {
			return true, nil
		}
		glog.V(5).Infof("DBG: fields is not a superset of args: fields=%v  args=%v", fields, args)
	}

	return false, nil
}

func isInterfaceUp(intf *net.Interface) bool {
	if intf == nil {
		return false
	}
	if intf.Flags&net.FlagUp != 0 {
		glog.V(4).Infof("Interface %v is up", intf.Name)
		return true
	}
	return false
}