func (p *dockerProvisioner) Destroy(app provision.App) error {
	containers, _ := listContainersByApp(app.GetName())
	go func(c []container) {
		var containersGroup sync.WaitGroup
		containersGroup.Add(len(containers))
		for _, c := range containers {
			go func(c container) {
				defer containersGroup.Done()
				err := removeContainer(&c)
				if err != nil {
					log.Error(err.Error())
				}
			}(c)
		}
		containersGroup.Wait()
		err := removeImage(assembleImageName(app.GetName()))
		if err != nil {
			log.Error(err.Error())
		}
	}(containers)
	r, err := getRouter()
	if err != nil {
		log.Errorf("Failed to get router: %s", err)
		return err
	}
	return r.RemoveBackend(app.GetName())
}

// handle is the function called by the queue handler on each message.
func handle(msg *queue.Message) {
	switch msg.Action {
	case regenerateApprc:
		if len(msg.Args) < 1 {
			log.Errorf("Error handling %q: this action requires at least 1 argument.", msg.Action)
			return
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Error(err.Error())
			return
		}
		err = app.SerializeEnvVars()
		if err != nil {
			log.Error(err.Error())
		}
	case BindService:
		err := bindUnit(msg)
		if err != nil {
			log.Error(err.Error())
			return
		}
	default:
		log.Errorf("Error handling %q: invalid action.", msg.Action)
	}
}

func (c *Client) BindUnit(instance *ServiceInstance, app bind.App, unit bind.Unit) error {
	log.Debugf("Calling bind of instance %q and %q unit at %q API",
		instance.Name, unit.GetIp(), instance.ServiceName)
	var resp *http.Response
	params := map[string][]string{
		"app-host":  {app.GetIp()},
		"unit-host": {unit.GetIp()},
	}
	resp, err := c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind", "POST", params)
	if err != nil {
		if m, _ := regexp.MatchString("", err.Error()); m {
			return fmt.Errorf("%s api is down.", instance.Name)
		}
		return err
	}
	switch resp.StatusCode {
	case http.StatusPreconditionFailed:
		return ErrInstanceNotReady
	case http.StatusNotFound:
		return ErrInstanceNotFoundInAPI
	}
	if resp.StatusCode > 299 {
		msg := fmt.Sprintf(`Failed to bind the instance "%s/%s" to the unit %q: %s`, instance.ServiceName, instance.Name, unit.GetIp(), c.buildErrorMessage(err, resp))
		log.Error(msg)
		return errors.New(msg)
	}
	return nil
}

func (c *Client) BindApp(instance *ServiceInstance, app bind.App) (map[string]string, error) {
	log.Debugf("Calling bind of instance %q and %q app at %q API",
		instance.Name, app.GetName(), instance.ServiceName)
	var resp *http.Response
	params := map[string][]string{
		"app-host": {app.GetIp()},
	}
	resp, err := c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind-app", "POST", params)
	if resp != nil && resp.StatusCode == http.StatusNotFound {
		resp, err = c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind", "POST", params)
	}
	if err != nil {
		log.Errorf(`Failed to bind app %q to service instance "%s/%s": %s`, app.GetName(), instance.ServiceName, instance.Name, err)
		return nil, fmt.Errorf("%s api is down.", instance.Name)
	}
	if err == nil && resp.StatusCode < 300 {
		var result map[string]string
		err = c.jsonFromResponse(resp, &result)
		if err != nil {
			return nil, err
		}
		return result, nil
	}
	switch resp.StatusCode {
	case http.StatusPreconditionFailed:
		return nil, ErrInstanceNotReady
	case http.StatusNotFound:
		return nil, ErrInstanceNotFoundInAPI
	}
	msg := fmt.Sprintf(`Failed to bind the instance "%s/%s" to the app %q: %s`, instance.ServiceName, instance.Name, app.GetName(), c.buildErrorMessage(err, resp))
	log.Error(msg)
	return nil, errors.New(msg)
}

func (c *Client) Status(instance *ServiceInstance) (string, error) {
	log.Debugf("Attempting to call status of service instance %q at %q api", instance.Name, instance.ServiceName)
	var (
		resp *http.Response
		err  error
	)
	url := "/resources/" + instance.GetIdentifier() + "/status"
	if resp, err = c.issueRequest(url, "GET", nil); err == nil {
		defer resp.Body.Close()
		switch resp.StatusCode {
		case http.StatusOK:
			var data []byte
			data, err = ioutil.ReadAll(resp.Body)
			return string(data), err
		case http.StatusAccepted:
			return "pending", nil
		case http.StatusNoContent:
			return "up", nil
		case http.StatusNotFound:
			return "not implemented for this service", nil
		case http.StatusInternalServerError:
			return "down", nil
		}
	}
	msg := "Failed to get status of instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
	log.Error(msg)
	return "", errors.New(msg)
}

// RecreateContainers relaunch all bs containers in the cluster for the given
// DockerProvisioner, logging progress to the given writer.
//
// It assumes that the given writer is thread safe.
func RecreateContainers(p DockerProvisioner, w io.Writer) error {
	cluster := p.Cluster()
	nodes, err := cluster.UnfilteredNodes()
	if err != nil {
		return err
	}
	errChan := make(chan error, len(nodes))
	wg := sync.WaitGroup{}
	log.Debugf("[bs containers] recreating %d containers", len(nodes))
	for i := range nodes {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			node := &nodes[i]
			pool := node.Metadata["pool"]
			log.Debugf("[bs containers] recreating container in %s [%s]", node.Address, pool)
			fmt.Fprintf(w, "relaunching bs container in the node %s [%s]\n", node.Address, pool)
			err := createContainer(node.Address, pool, p, true)
			if err != nil {
				msg := fmt.Sprintf("[bs containers] failed to create container in %s [%s]: %s", node.Address, pool, err)
				log.Error(msg)
				err = errors.New(msg)
				errChan <- err
			}
		}(i)
	}
	wg.Wait()
	close(errChan)
	return <-errChan
}

func (c *Client) Create(instance *ServiceInstance, user string) error {
	var err error
	var resp *http.Response
	params := map[string][]string{
		"name": {instance.Name},
		"user": {user},
		"team": {instance.TeamOwner},
	}
	if instance.PlanName != "" {
		params["plan"] = []string{instance.PlanName}
	}
	if instance.Description != "" {
		params["description"] = []string{instance.Description}
	}
	log.Debugf("Attempting to call creation of service instance for %q, params: %#v", instance.ServiceName, params)
	if resp, err = c.issueRequest("/resources", "POST", params); err == nil && resp.StatusCode < 300 {
		return nil
	}
	if resp.StatusCode == http.StatusConflict {
		return ErrInstanceAlreadyExistsInAPI
	}
	msg := "Failed to create the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
	log.Error(msg)
	return errors.New(msg)
}

func addLogs(ws *websocket.Conn) {
	var err error
	defer func() {
		data := map[string]interface{}{}
		if err != nil {
			data["error"] = err.Error()
			log.Error(err.Error())
		} else {
			data["error"] = nil
		}
		msg, _ := json.Marshal(data)
		ws.Write(msg)
		ws.Close()
	}()
	req := ws.Request()
	t := context.GetAuthToken(req)
	if t == nil {
		err = fmt.Errorf("wslogs: no token")
		return
	}
	if t.GetAppName() != app.InternalAppName {
		err = fmt.Errorf("wslogs: invalid token app name: %q", t.GetAppName())
		return
	}
	err = scanLogs(ws)
	if err != nil {
		return
	}
}

func (c *Client) BindApp(instance *ServiceInstance, app bind.App) (map[string]string, error) {
	log.Debugf("Calling bind of instance %q and %q app at %q API",
		instance.Name, app.GetName(), instance.ServiceName)
	var resp *http.Response
	params := map[string][]string{
		"app-host": {app.GetIp()},
	}
	resp, err := c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind-app", "POST", params)
	if resp != nil && resp.StatusCode == http.StatusNotFound {
		resp, err = c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind", "POST", params)
	}
	if err != nil {
		if m, _ := regexp.MatchString("", err.Error()); m {
			return nil, fmt.Errorf("%s api is down.", instance.Name)
		}
		return nil, err
	}
	if err == nil && resp.StatusCode < 300 {
		var result map[string]string
		err = c.jsonFromResponse(resp, &result)
		if err != nil {
			return nil, err
		}
		return result, nil
	}
	if resp.StatusCode == http.StatusPreconditionFailed {
		return nil, &errors.HTTP{Code: resp.StatusCode, Message: "You cannot bind any app to this service instance because it is not ready yet."}
	}
	msg := fmt.Sprintf("Failed to bind the instance %q to the app %q: %s", instance.Name, app.GetName(), c.buildErrorMessage(err, resp))
	log.Error(msg)
	return nil, &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
}

func (c *Client) BindUnit(instance *ServiceInstance, app bind.App, unit bind.Unit) error {
	log.Debugf("Calling bind of instance %q and %q unit at %q API",
		instance.Name, unit.GetIp(), instance.ServiceName)
	var resp *http.Response
	params := map[string][]string{
		"app-host":  {app.GetIp()},
		"unit-host": {unit.GetIp()},
	}
	resp, err := c.issueRequest("/resources/"+instance.GetIdentifier()+"/bind", "POST", params)
	if err != nil {
		if m, _ := regexp.MatchString("", err.Error()); m {
			return fmt.Errorf("%s api is down.", instance.Name)
		}
		return err
	}
	if resp.StatusCode == http.StatusPreconditionFailed {
		return &errors.HTTP{Code: resp.StatusCode, Message: "You cannot bind any app to this service instance because it is not ready yet."}
	}
	if resp.StatusCode > 299 {
		msg := fmt.Sprintf("Failed to bind the instance %q to the unit %q: %s", instance.Name, unit.GetIp(), c.buildErrorMessage(err, resp))
		log.Error(msg)
		return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
	}
	return nil
}

func (fn AdminRequiredHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	setVersionHeaders(w)
	defer func() {
		if r.Body != nil {
			r.Body.Close()
		}
	}()
	fw := io.FlushingWriter{ResponseWriter: w}
	header := r.Header.Get("Authorization")
	if header == "" {
		http.Error(&fw, "You must provide the Authorization header", http.StatusUnauthorized)
	} else if t, err := app.AuthScheme.Auth(header); err != nil {
		http.Error(&fw, "Invalid token", http.StatusUnauthorized)
	} else if user, err := t.User(); err != nil || !user.IsAdmin() {
		http.Error(&fw, "Forbidden", http.StatusForbidden)
	} else if err = fn(&fw, r, t); err != nil {
		code := http.StatusInternalServerError
		if e, ok := err.(*errors.HTTP); ok {
			code = e.Code
		}
		if fw.Wrote() {
			fmt.Fprintln(&fw, err)
		} else {
			http.Error(&fw, err.Error(), code)
		}
		log.Error(err.Error())
	}
}

// handle is the function called by the queue handler on each message.
func handle(msg *queue.Message) {
	switch msg.Action {
	case RegenerateApprcAndStart:
		fallthrough
	case regenerateApprc:
		if len(msg.Args) < 1 {
			log.Errorf("Error handling %q: this action requires at least 1 argument.", msg.Action)
			return
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Error(err.Error())
			return
		}
		err = app.SerializeEnvVars()
		if err != nil {
			log.Error(err.Error())
		}
		fallthrough
	case startApp:
		if msg.Action == regenerateApprc {
			break
		}
		if len(msg.Args) < 1 {
			log.Errorf("Error handling %q: this action requires at least 1 argument.", msg.Action)
		}
		app, err := ensureAppIsStarted(msg)
		if err != nil {
			log.Error(err.Error())
			return
		}
		err = app.Restart(ioutil.Discard)
		if err != nil {
			log.Errorf("Error handling %q. App failed to start:\n%s.", msg.Action, err)
			return
		}
	case BindService:
		err := bindUnit(msg)
		if err != nil {
			log.Error(err.Error())
			return
		}
	default:
		log.Errorf("Error handling %q: invalid action.", msg.Action)
	}
}

func getELBEndpoint() *elb.ELB {
	access, err := config.GetString("aws:access-key-id")
	if err != nil {
		log.Error(err.Error())
	}
	secret, err := config.GetString("aws:secret-access-key")
	if err != nil {
		log.Error(err.Error())
	}
	endpoint, err := config.GetString("juju:elb-endpoint")
	if err != nil {
		log.Error(err.Error())
	}
	auth := aws.Auth{AccessKey: access, SecretKey: secret}
	region := aws.Region{ELBEndpoint: endpoint}
	return elb.New(auth, region)
}

// ServerURL returns the URL to Gandalf API.
func ServerURL() string {
	server, err := config.GetString("git:api-server")
	if err != nil {
		log.Error("git:api-server config not found")
		panic(err)
	}
	return server
}

func addLogs(ws *websocket.Conn) {
	var err error
	defer func() {
		data := map[string]interface{}{}
		if err != nil {
			data["error"] = err.Error()
			log.Error(err.Error())
		} else {
			data["error"] = nil
		}
		msg, _ := json.Marshal(data)
		ws.Write(msg)
		ws.Close()
	}()
	req := ws.Request()
	t := context.GetAuthToken(req)
	if t == nil {
		err = fmt.Errorf("wslogs: no token")
		return
	}
	if t.GetAppName() != app.InternalAppName {
		err = fmt.Errorf("wslogs: invalid token app name: %q", t.GetAppName())
		return
	}
	dispatcher := app.NewlogDispatcher()
	scanner := bufio.NewScanner(ws)
	for scanner.Scan() {
		var entry app.Applog
		data := bytes.TrimSpace(scanner.Bytes())
		if len(data) == 0 {
			continue
		}
		err = json.Unmarshal(data, &entry)
		if err != nil {
			dispatcher.Stop()
			err = fmt.Errorf("wslogs: parsing log line %q: %s", string(data), err)
			return
		}
		err = dispatcher.Send(&entry)
		if err != nil {
			// Do not disconnect by returning here, dispatcher will already
			// retry db connection and we gain nothing by ending the WS
			// connection.
			log.Errorf("wslogs: error storing log: %s", err)
		}
	}
	err = dispatcher.Stop()
	if err != nil {
		err = fmt.Errorf("wslogs: error storing log: %s", err)
		return
	}
	err = scanner.Err()
	if err != nil {
		err = fmt.Errorf("wslogs: waiting for log data: %s", err)
		return
	}
}

func addLogs(ws *websocket.Conn) {
	var err error
	defer func() {
		data := map[string]interface{}{}
		if err != nil {
			data["error"] = err.Error()
			log.Error(err.Error())
		} else {
			data["error"] = nil
		}
		msg, _ := json.Marshal(data)
		ws.Write(msg)
		ws.Close()
	}()
	req := ws.Request()
	t := context.GetAuthToken(req)
	if t == nil {
		err = fmt.Errorf("wslogs: no token")
		return
	}
	if t.GetAppName() != app.InternalAppName {
		err = fmt.Errorf("wslogs: invalid token app name: %q", t.GetAppName())
		return
	}
	logCh, errCh := app.LogReceiver()
	scanner := bufio.NewScanner(ws)
	for scanner.Scan() {
		var entry app.Applog
		data := bytes.TrimSpace(scanner.Bytes())
		if len(data) == 0 {
			continue
		}
		err = json.Unmarshal(data, &entry)
		if err != nil {
			close(logCh)
			err = fmt.Errorf("wslogs: parsing log line %q: %s", string(data), err)
			return
		}
		select {
		case logCh <- &entry:
		case err := <-errCh:
			close(logCh)
			err = fmt.Errorf("wslogs: storing log: %s", err)
			return
		}
	}
	close(logCh)
	err = scanner.Err()
	if err != nil {
		err = fmt.Errorf("wslogs: waiting for log data: %s", err)
		return
	}
	err = <-errCh
	if err != nil {
		err = fmt.Errorf("wslogs: storing log: %s", err)
	}
}

// RemoveUnit removes a unit by its Name.
func (app *App) RemoveUnit(name string) error {
	unit, err := app.findUnitByName(name)
	if err != nil {
		return err
	}
	if err = Provisioner.RemoveUnit(app, unit.Name); err != nil {
		log.Error(err.Error())
	}
	return app.unbindUnit(unit)
}

// RemoveUnit removes a unit by its InstanceId or Name.
//
// Will search first by InstanceId, if no instance is found, then tries to
// search using the unit name, then calls the removal function from provisioner
//
// Returns an error in case of failure.
func (app *App) RemoveUnit(id string) error {
	unit, i, err := app.findUnitByID(id)
	if err != nil {
		return err
	}
	if err = Provisioner.RemoveUnit(app, unit.GetName()); err != nil {
		log.Error(err.Error())
	}
	return app.removeUnitByIdx(i, unit)
}

func (c *Client) Destroy(instance *ServiceInstance) error {
	log.Debugf("Attempting to call destroy of service instance %q at %q api", instance.Name, instance.ServiceName)
	resp, err := c.issueRequest("/resources/"+instance.GetIdentifier(), "DELETE", nil)
	if err == nil && resp.StatusCode > 299 {
		msg := "Failed to destroy the instance " + instance.Name + ": " + c.buildErrorMessage(err, resp)
		log.Error(msg)
		return &errors.HTTP{Code: http.StatusInternalServerError, Message: msg}
	}
	return err
}

func ensureContainersStarted(p DockerProvisioner, w io.Writer, relaunch bool, nodes ...cluster.Node) error {
	if w == nil {
		w = ioutil.Discard
	}
	confNames, err := scopedconfig.FindAllScopedConfigNames(nodeContainerCollection)
	if err != nil {
		return err
	}
	if len(nodes) == 0 {
		nodes, err = p.Cluster().UnfilteredNodes()
		if err != nil {
			return err
		}
	}
	errChan := make(chan error, len(nodes))
	wg := sync.WaitGroup{}
	log.Debugf("[node containers] recreating %d containers", len(nodes))
	recreateContainer := func(node *cluster.Node, confName string) {
		defer wg.Done()
		pool := node.Metadata["pool"]
		containerConfig, confErr := LoadNodeContainer(pool, confName)
		if confErr != nil {
			errChan <- confErr
			return
		}
		log.Debugf("[node containers] recreating container %q in %s [%s]", confName, node.Address, pool)
		fmt.Fprintf(w, "relaunching node container %q in the node %s [%s]\n", confName, node.Address, pool)
		confErr = containerConfig.create(node.Address, pool, p, relaunch)
		if confErr != nil {
			msg := fmt.Sprintf("[node containers] failed to create container in %s [%s]: %s", node.Address, pool, confErr)
			log.Error(msg)
			errChan <- errors.New(msg)
		}
	}
	for i := range nodes {
		wg.Add(1)
		go func(node *cluster.Node) {
			defer wg.Done()
			for j := range confNames {
				wg.Add(1)
				go recreateContainer(node, confNames[j])
			}
		}(&nodes[i])
	}
	wg.Wait()
	close(errChan)
	var allErrors []string
	for err = range errChan {
		allErrors = append(allErrors, err.Error())
	}
	if len(allErrors) == 0 {
		return nil
	}
	return fmt.Errorf("multiple errors: %s", strings.Join(allErrors, ", "))
}