def create_all_boot_snapshots(self):
        v = []

        v.append(('kubectl', ''))

        for i in [0,1,2,3,4,5]:
            v.append(('db', i))

        #for name in self.dev_instances():
        #    node = name.split('v')[1]
        #    v.append(('dev', node))

        for i in [0,1,2]:
            v.append(('web',i))
        v.append(('admin',0))
        log("snapshotting storage machine boot images")
        for i in [0,1,2,3,4,5]:
            v.append(('storage', i))
        log("snapshotting compute machine boot images")
        for i in [0,1,2,3,4,5,6,7]:
            v.append(('compute', i))

        errors = []
        log("snapshotting boot images: %s"%v)
        for prefix, node in v:
            try:
                log('snapshotting %s%s'%(prefix, node))
                self.create_boot_snapshot(node=node, prefix=prefix, zone='us-central1-c', devel=False)
            except Exception, mesg:
                errors.append(mesg)
                log("WARNING: issue making snapshot -- %s", mesg)

    def create_data_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
        """
        Snapshot the data disk on the given machine.  Typically used for
        backing up very important data.
        """
        zone = self.expand_zone(zone)
        instance_name = self.instance_name(node, prefix, zone, devel=devel)
        info = json.loads(cmd(['gcloud', 'compute', 'instances', 'describe',
                               instance_name, '--zone', zone, '--format=json'], verbose=0))

        errors = []
        for disk in info['disks']:
            # ignore boot disks (would be True)
            if disk.get('boot', False):
                continue
            # ignore read-only disks (like for globally mounted data volumes)
            # they would be snapshotted manually
            if disk.get('mode', 'READ_WRITE') == 'READ_ONLY':
                continue

            src = disk['source'].split('/')[-1]
            if 'swap' in src: continue
            if 'tmp' in src: continue
            target = 'data-%s-%s'%(src, time.strftime(TIMESTAMP_FORMAT))
            log("%s --> %s", src, target)
            try:
                cmd(['gcloud', 'compute', 'disks', 'snapshot',
                     '--project', self.project,
                    src,
                     '--snapshot-names', target,
                     '--zone', zone], system=True)
            except Exception, mesg:
                log("WARNING: issue making snapshot %s -- %s", target, mesg)
                errors.append(mesg)

 def instance_costs(self):
     cost = cost_upper = 0
     n_compute = 0
     n_smc = 0
     n_other = 0
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         zone         = v[1]
         machine_type = v[2]
         status       = v[-1]
         if v[0].startswith('compute'):
             n_compute += 1
         elif v[0].startswith('smc'):
             n_smc += 1
         else:
             n_other += 1
         if status == "RUNNING":
             t = machine_type.split('-')
             if len(t) == 3:
                 b = '-'.join(t[:2])
                 cpus = int(t[2])
             else:
                 b = machine_type
                 cpus = 1
             cost += PRICING[b+'-month'] * cpus * PRICING[zone.split('-')[0]]
             cost_upper += PRICING[b+'-hour'] *30.5*24* cpus * PRICING[zone.split('-')[0]]
     log("INSTANCES    : compute=%s, smc=%s, other=%s: %s/month (or %s/month with sustained use)",
         n_compute, n_smc, n_other, money(cost_upper), money(cost))
     return cost_upper

 def network_costs(self):
     # These are estimates based on usage during March and April.  May be lower in future
     # do to moving everything to GCE.  Not sure.
     us = 700; aus = 150; china = 20
     costs = 700 * PRICING['egress'] + 150*PRICING['egress-australia'] + 20*PRICING['egress-china']
     log("NETWORK      : %8s/month -- approx. %sGB Americas, %sGB EUR, %sGB CHINA", money(costs), us, aus, china)
     return costs

    def create_data_snapshot(self, node, prefix, zone='us-central1-c', devel=False):
        """
        Snapshot the data disk on the given machine.  Typically used for
        backing up very important data.
        """
        zone = self.expand_zone(zone)
        instance_name = self.instance_name(node, prefix, zone, devel=devel)
        info = json.loads(cmd(['gcloud', 'compute', 'instances', 'describe',
                               instance_name, '--zone', zone, '--format=json'], verbose=0))

        errors = []
        for disk in info['disks']:
            if disk.get('boot', False):
                continue
            src = disk['deviceName']
            if 'swap' in src: continue
            target = 'data-%s-%s'%(src, time.strftime(TIMESTAMP_FORMAT))
            log("%s --> %s", src, target)
            try:
                cmd(['gcloud', 'compute', 'disks', 'snapshot',
                     '--project', self.project,
                    src,
                     '--snapshot-names', target,
                     '--zone', zone], system=True)
            except Exception, mesg:
                log("WARNING: issue making snapshot %s -- %s", target, mesg)
                errors.append(mesg)

    def create_all_data_snapshots(self, zone='us-central1-c'):
        for i in range(6):
            log("snapshotting storage%s storage data"%i)
            self.create_data_snapshot(node=i, prefix='storage', zone=zone, devel=False)

        log("snapshotting live user data")
        for n in self.compute_nodes(zone):
            self.create_data_snapshot(node=n, prefix='compute', zone=zone, devel=False)

 def costs(self):
     costs = {}
     total = 0
     for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
         costs[t] = getattr(self, '%s_costs'%t)()
         total += costs[t]
     log("TOTAL        : %s/month", money(total))
     return costs

def setup_projects_path():
    log("create paths")
    cmd("sudo mkdir -p /projects")
    cmd("sudo chmod a+x /projects")
    cmd("sudo touch /projects/snapshots; sudo chmod a+r /projects/snapshots")
    cmd("sudo mkdir -p /projects/conf")
    cmd("sudo chown salvus. /projects/conf")
    cmd("sudo mkdir -p /projects/sagemathcloud")
    cmd("sudo rsync -LrxH --delete /home/salvus/salvus/salvus/local_hub_template/ /projects/sagemathcloud/")

 def delete_devel_instances(self):
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         name         = v[0]
         if '-devel-' in name:
             zone         = v[1]
             status       = v[-1]
             log("deleting devel instance: %s"%name)
             cmd(['gcloud', 'compute', 'instances', 'delete', '--zone', zone, name], system=True)

 def instance_costs(self):
     cost_lower = cost_upper = 0
     n_compute = 0
     n_web = 0
     n_db = 0
     n_other = 0
     n_dev =0
     n_admin =0
     n_storage =0
     n_preempt = 0
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         zone         = v[1]
         machine_type = v[2]
         status       = v[-1]
         if status != 'RUNNING':
             continue
         if len(v) == 7:
             preempt = (v[3] == 'true')
             n_preempt += 1
         else:
             preempt = False
         if v[0].startswith('compute'):
             n_compute += 1
         elif v[0].startswith('web'):
             n_web += 1
         elif v[0].startswith('db'):
             n_db += 1
         elif v[0].startswith('dev'):
             n_dev += 1
         elif v[0].startswith('admin'):
             n_admin += 1
         elif v[0].startswith('storage'):
             n_storage += 1
         else:
             n_other += 1
         t = machine_type.split('-')
         if len(t) == 3:
             b = '-'.join(t[:2])
             cpus = int(t[2])
         else:
             b = machine_type
             cpus = 1
         if b == 'custom':
             print("warning -custom machine types not supported; skipping ", x)
             continue
         if preempt:
             pricing_hour  = PRICING[b+'-hour-pre']
             pricing_month = pricing_hour*24*30.5
         else:
             pricing_hour  = PRICING[b+'-hour']
             pricing_month = PRICING[b+'-month']
         cost_lower += pricing_month * cpus * PRICING[zone.split('-')[0]]
         cost_upper += pricing_hour *30.5*24* cpus * PRICING[zone.split('-')[0]]
     log("INSTANCES    : %8s/month -- (or %8s/month without sustained!); compute=%s, web=%s, db=%s, dev=%s, admin=%s, storage=%s, other=%s (preempt=%s)",
          money(cost_lower), money(cost_upper), n_compute, n_web, n_db, n_dev, n_admin, n_storage,  n_other, n_preempt)
     return {'lower':cost_lower, 'upper':cost_upper}

 def start_devel_instances(self):
     for x in cmd(['gcloud', 'compute', 'instances', 'list']).splitlines()[1:]:
         v = x.split()
         name         = v[0]
         if '-devel-' in name:
             zone         = v[1]
             status       = v[-1]
             if status == "TERMINATED":
                 log("starting %s"%name)
                 cmd(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name])

def create_data_secrets():
    cmd("mkdir -p %s"%SECRETS)
    log("sendgrid fake password (will not work)")
    cmd("echo 'will-not-work' > %s/sendgrid_email_password"%SECRETS)
    log("generate cassandra passwords")
    cmd("mkdir -p %s/cassandra"%SECRETS)
    cmd("makepasswd -q > %s/cassandra/hub"%SECRETS)
    cmd("makepasswd -q > %s/cassandra/salvus"%SECRETS)
    cmd("mkdir -p %s/sagemath.com"%SECRETS)
    cmd("yes US | openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem -nodes -days 10000 && cat key.pem cert.pem > %s/sagemath.com/nopassphrase.pem"%SECRETS)

def delete_secrets():
    log("delete any possible sensitive info from the production install")
    log("wipe root ssh keys")
    cmd("sudo rm -f /root/.ssh/id_rsa /root/.ssh/id_rsa.pub")
    log("wipe salvus ssh keys")
    cmd("sudo rm -rf /home/salvus/.ssh/id_rsa*")
    log("wipe salvus secrets")
    cmd("sudo rm -rf /home/salvus/salvus/salvus/data/secrets/")
    log("wipe production logs")
    cmd("sudo rm -rf /home/salvus/logs/*")

 def create_all_boot_snapshots(self):
     log("snapshotting storage boot image")
     self.create_boot_snapshot(node=0, prefix='storage', zone='us-central1-c', devel=False)
     log("snapshotting backup boot image")
     self.create_boot_snapshot(node=0, prefix='backup', zone='us-central1-c', devel=False)
     log("snapshotting admin boot image")
     self.create_boot_snapshot(node='',prefix='admin', zone='us-central1-c', devel=False)
     log("snapshotting SMC server boot image")
     self.create_boot_snapshot(node=0, prefix='smc', zone='us-central1-c', devel=False)
     log("snapshotting compute machine boot image")
     self.create_boot_snapshot(node=0, prefix='compute', zone='us-central1-c', devel=False)

    def create_dev(self, node, zone='us-central1-c', machine_type='n1-standard-1', size=30, preemptible=True, address=''):
        zone = self.expand_zone(zone)
        name = self.instance_name(node=node, prefix='dev', zone=zone)

        log("creating %sGB hard disk root filesystem image based on last smc snapshot", size)
        try:
            cmd(['gcloud', 'compute', '--project', self.project, 'disks', 'create', name,
                 '--zone', zone, '--source-snapshot', self.newest_snapshot('smc'),
                 '--size', size, '--type', 'pd-standard'])
        except Exception, mesg:
            if 'already exists' not in str(mesg):
                raise

    def gcs_costs(self):
        # usage based on running "time gsutil du -sch" every once in a while, since it takes
        # quite a while to run.
        cassandra = 200
        database_backup	= 200
        gb_archive = 650  # delete in a few weeks...
        projects_backup = 1500

        usage = (database_backup + gb_archive + projects_backup)
        costs = usage * PRICING['gcs-nearline']
        log("CLOUD STORAGE: approx. %sGB nearline: %s/month", usage, money(costs))
        return costs

 def costs(self):
     costs = {}
     total_lower = 0
     total_upper = 0
     for t in ['snapshot', 'disk', 'instance', 'network', 'gcs']:
         costs[t] = getattr(self, '%s_costs'%t)()
         if isinstance(costs[t], dict):
             total_lower += costs[t]['lower']
             total_upper += costs[t]['upper']
         else:
             total_lower += costs[t]
             total_upper += costs[t]
     log("TOTAL        : %8s/month -- up to as worse as %8s/month without sustained", money(total_lower), money(total_upper))

def init_cassandra_users():
    pw_hub = open("%s/cassandra/hub"%SECRETS).read()
    cmd("""echo "CREATE USER hub WITH PASSWORD '%s' SUPERUSER;" | cqlsh localhost -u cassandra -p cassandra"""%pw_hub, verbose=0)
    rc = "%s/.cqlshrc"%os.environ['HOME']
    log("writing %s", rc)
    open(rc, 'w').write("""
[authentication]
username=hub
password=%s
"""%pw_hub)
    pw_salvus = open("%s/cassandra/salvus"%SECRETS).read()
    cmd("""echo "CREATE USER salvus WITH PASSWORD '%s' SUPERUSER;" | cqlsh localhost -u cassandra -p cassandra"""%pw_salvus, verbose=0)
    cmd("""echo "ALTER USER cassandra WITH PASSWORD '%s';" | cqlsh localhost -u cassandra -p cassandra"""%pw_hub, verbose=0)

 def autostart(self, instance):
     """
     Ensure that each instance in the input is running.
     """
     for x in cmd(['gcloud', 'compute', 'instances', 'list'], verbose=0).splitlines()[1:]:
         v = x.split()
         if len(v) > 2 and v[-1] != 'RUNNING':
             name = v[0]; zone = v[1]
             for x in instance:
                 if name.startswith(x):
                     log("Starting %s... at %s", name, time.asctime())
                     cmd(' '.join(['gcloud', 'compute', 'instances', 'start', '--zone', zone, name]) + '&', system=True)
                     break

    def gcs_costs(self):
        # usage based on running "time gsutil du -sch" every once in a while, since it takes
        # quite a while to run.
        smc_db_backup = 50
        smc_projects_backup = 1300
        # This takes about 15 minutes and gives the usage above
        #     time gsutil du -sch gs://smc-projects-bup
        #     time gsutil du -sch gs://smc-db-backup

        usage = (smc_db_backup + smc_projects_backup)
        costs = usage * PRICING['gcs-nearline']
        log("CLOUD STORAGE: %8s/month -- approx. %sGB nearline", money(costs), usage)
        return costs