def create_data_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
        """
        Snapshot the data disk on the given machine.  Typically used for
        backing up very important data.
        """
        zone = self.expand_zone(zone)
        instance_name = self.instance_name(node, prefix, zone, devel=devel)
        info = json.loads(cmd(['gcloud', 'compute', 'instances', 'describe',
                               instance_name, '--zone', zone, '--format=json'], verbose=0))

        errors = []
        for disk in info['disks']:
            if disk.get('boot', False):
                continue
            src = disk['deviceName']
            if 'swap' in src: continue
            target = 'data-%s-%s'%(src, time.strftime(TIMESTAMP_FORMAT))
            log("%s --> %s", src, target)
            try:
                cmd(['gcloud', 'compute', 'disks', 'snapshot',
                     '--project', self.project,
                    src,
                     '--snapshot-names', target,
                     '--zone', zone], system=True)
            except Exception, mesg:
                log("WARNING: issue making snapshot %s -- %s", target, mesg)
                errors.append(mesg)

    def create_data_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
        """
        Snapshot the data disk on the given machine.  Typically used for
        backing up very important data.
        """
        zone = self.expand_zone(zone)
        instance_name = self.instance_name(node, prefix, zone, devel=devel)
        info = json.loads(cmd(['gcloud', 'compute', 'instances', 'describe',
                               instance_name, '--zone', zone, '--format=json'], verbose=0))

        errors = []
        for disk in info['disks']:
            # ignore boot disks (would be True)
            if disk.get('boot', False):
                continue
            # ignore read-only disks (like for globally mounted data volumes)
            # they would be snapshotted manually
            if disk.get('mode', 'READ_WRITE') == 'READ_ONLY':
                continue

            src = disk['source'].split('/')[-1]
            if 'swap' in src: continue
            if 'tmp' in src: continue
            target = 'data-%s-%s'%(src, time.strftime(TIMESTAMP_FORMAT))
            log("%s --> %s", src, target)
            try:
                cmd(['gcloud', 'compute', 'disks', 'snapshot',
                     '--project', self.project,
                    src,
                     '--snapshot-names', target,
                     '--zone', zone], system=True)
            except Exception, mesg:
                log("WARNING: issue making snapshot %s -- %s", target, mesg)
                errors.append(mesg)

def start_cassandra():
    log("start_cassandra...")
    services = admin.Services('conf/deploy_devel/', password='')
    services.start('cassandra')
    cmd("ln -sf %s/data/cassandra-0/logs/system.log %s/logs/cassandra.log"%(SALVUS_ROOT, os.environ['HOME']))
    log("cassandra started")
    log("waiting 30 seconds...")
    import time; time.sleep(30)

def setup_quota():
    log("quota packages")
    cmd("sudo apt-get install -y libatlas3gf-base liblapack-dev  quota quotatool linux-image-extra-virtual cgroup-lite cgmanager-utils cgroup-bin libpam-cgroup cgmanager cgmanager-utils  cgroup-bin smem", system=True)
    log("quota stuff")
    cmd("echo 'LABEL=cloudimg-rootfs /        ext4   defaults,usrquota       0 0' | sudo tee /etc/fstab")
    cmd("sudo mount -o remount /")
    log("initializing quota, which will take a while")
    cmd("sudo quotacheck -fucm /")
    cmd("sudo quotaon /")

 def delete_devel_instances(self):
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         name         = v[0]
         if '-devel-' in name:
             zone         = v[1]
             status       = v[-1]
             log("deleting devel instance: %s"%name)
             cmd(['gcloud', 'compute', 'instances', 'delete', '--zone', zone, name], system=True)

 def start_devel_instances(self):
     for x in cmd(['gcloud', 'compute', 'instances', 'list']).splitlines()[1:]:
         v = x.split()
         name         = v[0]
         if '-devel-' in name:
             zone         = v[1]
             status       = v[-1]
             if status == "TERMINATED":
                 log("starting %s"%name)
                 cmd(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name])

 def create_boot_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
     """
     Snapshot the boot disk on the give machine.  Typically used for
     replicating configuration.
     """
     zone = self.expand_zone(zone)
     instance_name = self.instance_name(node, prefix, zone, devel=devel)
     snapshot_name = "%s%s-%s"%(prefix, node, time.strftime(TIMESTAMP_FORMAT))
     cmd(['gcloud', 'compute', 'disks', 'snapshot', '--project', self.project,
         instance_name,
         '--snapshot-names', snapshot_name,
         '--zone', zone], system=True)

    def create_dev(self, node, zone='us-central1-c', machine_type='n1-standard-1', size=30, preemptible=True, address=''):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='dev', zone=zone)

        log("creating %sGB hard disk root filesystem image based on last smc snapshot", size)
        try:
            cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
                 '--size', size, '--type', 'pd-standard'])
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise

 def autostart(self, instance):
     """
     Ensure that each instance in the input is running.
     """
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         if len(v) > 2 and v[-1] != 'RUNNING':
             name = v[0]; zone = v[1]
             for x in instance:
                 if name.startswith(x):
                     log("Starting %s... at %s", name, time.asctime())
                     cmd(' '.join(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name]) + '&', system=True)
                     break

    def _create_storage_server(self, node, zone, machine_type,
                               disk_size, network, devel):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='storage', zone=zone, devel=devel)
        disk_name = "%s-projects"%name

        log("creating hard disk root filesystem image")
        try:
            cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('storage'),
                 '--type', 'pd-standard'])
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise

 def devel_etc_hosts(self):
     hosts = []
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         name         = v[0]
         if '-devel-' in name:
             i = name.find('-devel')
             hosts.append("%s %s %s"%(v[4], v[0], v[0][:i+6]))
     if hosts:
         print "\n".join(hosts)
         x = open("/etc/hosts").readlines()
         y = [a.strip() for a in x if '-devel-' not in a]
         open('/tmp/hosts','w').write('\n'.join(y+hosts))
         cmd("sudo cp -v /etc/hosts /etc/hosts.0 && sudo cp -v /tmp/hosts /etc/hosts", system=True)

    def _create_smc_server(self, node, zone='us-central1-c', machine_type='n1-highmem-2',
                          disk_size=100, network='default', devel=False):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='smc', zone=zone, devel=devel)
        disk_name = "%s-cassandra"%name

        log("creating hard disk root filesystem image")
        try:
            cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
                 '--type', 'pd-standard'])
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise

 def set_metadata(self, prefix=''):
     if not prefix:
         for p in ['smc', 'compute', 'admin', 'storage']:
             self.set_metadata(p)
         return
     names = []
     for x in cmd(['gcloud', 'compute', 'instances', 'list']).splitlines()[1:]:
         v = x.split()
         if v[-1] != 'RUNNING':
             continue
         name = v[0]
         if name.startswith(prefix) and 'devel' not in name: #TODO
             names.append(name)
     names = ' '.join(names)
     cmd(['gcloud', 'compute', 'project-info', 'add-metadata', '--metadata', '%s-servers=%s'%(prefix, names)])

 def dev_instances(self):
     a = []
     for x in cmd(['gcloud', 'compute', 'instances', 'list']).splitlines()[1:]:
         name = x.split()[0]
         if name.startswith('dev'):
             a.append(name)
     return a

 def compute_nodes(self, zone='us-central1-c'):
     # names of the compute nodes in the given zone, with the zone postfix and compue prefix removed.
     n = len("compute")
     def f(name):
         return name[n:name.rfind('-')]
     info = json.loads(cmd(['gcloud', 'compute', 'instances', 'list', '-r', '^compute.*', '--format=json'], verbose=0))
     return [f(x['name']) for x in info if x['zone'] == zone and f(x['name'])]

 def instance_costs(self):
     cost = cost_upper = 0
     n_compute = 0
     n_smc = 0
     n_other = 0
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         zone         = v[1]
         machine_type = v[2]
         status       = v[-1]
         if v[0].startswith('compute'):
             n_compute += 1
         elif v[0].startswith('smc'):
             n_smc += 1
         else:
             n_other += 1
         if status == "RUNNING":
             t = machine_type.split('-')
             if len(t) == 3:
                 b = '-'.join(t[:2])
                 cpus = int(t[2])
             else:
                 b = machine_type
                 cpus = 1
             cost += PRICING[b+'-month'] * cpus * PRICING[zone.split('-')[0]]
             cost_upper += PRICING[b+'-hour'] *30.5*24* cpus * PRICING[zone.split('-')[0]]
     log("INSTANCES    : compute=%s, smc=%s, other=%s: %s/month (or %s/month with sustained use)",
         n_compute, n_smc, n_other, money(cost_upper), money(cost))
     return cost_upper

def delete_secrets():
    log("delete any possible sensitive info from the production install")
    log("wipe root ssh keys")
    cmd("sudo rm -f /root/.ssh/id_rsa /root/.ssh/id_rsa.pub")
    log("wipe salvus ssh keys")
    cmd("sudo rm -rf /home/salvus/.ssh/id_rsa*")
    log("wipe salvus secrets")
    cmd("sudo rm -rf /home/salvus/salvus/salvus/data/secrets/")
    log("wipe production logs")
    cmd("sudo rm -rf /home/salvus/logs/*")

 def instance_costs(self):
     cost_lower = cost_upper = 0
     n_compute = 0
     n_web = 0
     n_db = 0
     n_other = 0
     n_dev =0
     n_admin =0
     n_storage =0
     n_preempt = 0
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         zone         = v[1]
         machine_type = v[2]
         status       = v[-1]
         if status != 'RUNNING':
             continue
         if len(v) == 7:
             preempt = (v[3] == 'true')
             n_preempt += 1
         else:
             preempt = False
         if v[0].startswith('compute'):
             n_compute += 1
         elif v[0].startswith('web'):
             n_web += 1
         elif v[0].startswith('db'):
             n_db += 1
         elif v[0].startswith('dev'):
             n_dev += 1
         elif v[0].startswith('admin'):
             n_admin += 1
         elif v[0].startswith('storage'):
             n_storage += 1
         else:
             n_other += 1
         t = machine_type.split('-')
         if len(t) == 3:
             b = '-'.join(t[:2])
             cpus = int(t[2])
         else:
             b = machine_type
             cpus = 1
         if b == 'custom':
             print("warning -custom machine types not supported; skipping ", x)
             continue
         if preempt:
             pricing_hour  = PRICING[b+'-hour-pre']
             pricing_month = pricing_hour*24*30.5
         else:
             pricing_hour  = PRICING[b+'-hour']
             pricing_month = PRICING[b+'-month']
         cost_lower += pricing_month * cpus * PRICING[zone.split('-')[0]]
         cost_upper += pricing_hour *30.5*24* cpus * PRICING[zone.split('-')[0]]
     log("INSTANCES    : %8s/month -- (or %8s/month without sustained!); compute=%s, web=%s, db=%s, dev=%s, admin=%s, storage=%s, other=%s (preempt=%s)",
          money(cost_lower), money(cost_upper), n_compute, n_web, n_db, n_dev, n_admin, n_storage,  n_other, n_preempt)
     return {'lower':cost_lower, 'upper':cost_upper}

    def _create_compute_server(self, node, zone='us-central1-c',
                              machine_type='n1-highmem-4', network='default',
                              projects_ssd=False, base_ssd=False,
                               projects_size=150,
                              devel=False):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='compute', zone=zone, devel=devel)

        log("creating root filesystem image")
        try:
            opts = ['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('compute')]
            if base_ssd:
                opts.extend(['--type', 'pd-ssd'])
            cmd(opts)
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise
            log("%s already exists", name)

    def delete_all_old_snapshots(self, max_age_days=7, quiet=False):
        snapshots = [x.split()[0] for x in cmd(['gcloud', 'compute', 'snapshots', 'list']).splitlines()[1:]]
        log("snapshots=%s", snapshots)
        # restrict to snapshots that end with a timestamp
        # and for each restructure by base
        w = {}
        n = len('2015-05-03-081013')
        for s in snapshots:
            try:
                time.strptime(s[-n:], TIMESTAMP_FORMAT)
                base = s[:-n]
                if base in w:
                    w[base].append(s[-n:])
                else:
                    w[base] = [s[-n:]]
            except: pass
        print w

        # now decide what to delete
        to_delete = []
        cutoff = time.strftime(TIMESTAMP_FORMAT, time.gmtime(time.time()-60*60*24*max_age_days))
        for base in w:
            v = w[base]
            v.sort()
            if len(v) <= 1 or v[0] >= cutoff:
                # definitely don't delete last one or if all are new
                continue
            for x in v:
                if x < cutoff:
                    to_delete.append(base + x)

        if len(to_delete) == 0:
            log("no old snapshots to delete")
        else:
            log("deleting these snapshots: %s", to_delete)
            a = ['gcloud', 'compute', 'snapshots', 'delete']
            if quiet:
                a.append("--quiet")
            cmd(a + to_delete, system=True)