def step_impl(context):
    sh.mkdir("tmp").wait()
    for row in context.table:
        sh.dd(
            "if=/dev/zero", "of=tmp/" + row["name"], "bs=1048576",
            "count=" + row["count"]
        )

def pull_file_from_device(serial_num, file_path, file_name, output_dir):
    if not os.path.exists(output_dir):
        sh.mkdir("-p", output_dir)
    output_path = "%s/%s" % (output_dir, file_path)
    if os.path.exists(output_path):
        sh.rm('-rf', output_path)
    adb_pull(file_path + '/' + file_name, output_dir, serial_num)

def copy_assets():
    """copy assets for static serving"""
    proj()

    print(". copying assets ...")

    copy_patterns = {
        "dist": ["./static/lib/jquery-1.8.3.min.js"] +
        sh.glob("./static/config/*.json") +
        sh.glob("./static/fragile-min.*"),

        "dist/font": sh.glob("./static/lib/awesome/font/*"),
        "dist/svg": sh.glob("./static/svg/*.svg"),
        "dist/img": sh.glob("./static/img/*.*") or [],
        
        "dist/docs/assets": sh.glob("./docs/assets/*.*") or [],
    }

    for dst, copy_files in copy_patterns.items():
        if not os.path.exists(dst):
            sh.mkdir("-p", dst)

        for c_file in copy_files:
            print "... copying", c_file, dst
            sh.cp("-r", c_file, dst)

    wa_cache = "./dist/.webassets-cache"

    if os.path.exists(wa_cache):
        sh.rm("-r", wa_cache)

def ensure_syncer_dir():
    if path.isdir(syncer_dir):
        return

    username = input('GitHub username: ')
    password = getpass.getpass('GitHub password: ')
    repo_exists = github.check_repo_exists(username, SYNCER_REPO_NAME)

    if not repo_exists:
        print("Creating new repo in GitHub")
        github.create_public_repo(username, password, SYNCER_REPO_NAME)

    print("Cloning GitHub repo.")
    sh.git('clone', 'https://%s:%[email protected]/%s/%s.git' % (username, password, username, SYNCER_REPO_NAME), syncer_dir)

    needs_commit = False
    sh.cd(syncer_dir)
    if not path.isfile(path('manifest.json')):
        sh.touch('manifest.json')
    
    if not path.isdir(path('content')):
        sh.mkdir('content')

    if not path.isdir(path('backup')):
        sh.mkdir('backup')

    if not path.isfile(path('.gitignore')):
        needs_commit = True
        with open('.gitignore', 'w') as gitignore_file:
            gitignore_file.write('backup')

    if needs_commit:
        sh.git('add', '-A')
        sh.git('commit', '-m', 'Setting up scaffolding.')

def build_opencv():
    sh.pip.install("numpy")
    clone_if_not_exists("opencv", "https://github.com/PolarNick239/opencv.git", branch="stable_3.0.0)
    clone_if_not_exists("opencv_contrib", "https://github.com/PolarNick239/opencv_contrib.git", branch="stable_3.0.0")
    sh.rm("-rf", "build")
    sh.mkdir("build")
    sh.cd("build")
    python_path = pathlib.Path(sh.pyenv.which("python").stdout.decode()).parent.parent
    version = "{}.{}".format(sys.version_info.major, sys.version_info.minor)
    sh.cmake(
        "..",
        "-DCMAKE_BUILD_TYPE=RELEASE",
        "-DCMAKE_INSTALL_PREFIX={}/usr/local".format(python_path),
        "-DWITH_CUDA=OFF",
        "-DWITH_FFMPEG=OFF",
        "-DINSTALL_C_EXAMPLES=OFF",
        "-DBUILD_opencv_legacy=OFF",
        "-DBUILD_NEW_PYTHON_SUPPORT=ON",
        "-DBUILD_opencv_python3=ON",
        "-DOPENCV_EXTRA_MODULES_PATH=~/opencv_contrib-3.4.1/modules",
        "-DBUILD_EXAMPLES=ON",
        "-DPYTHON_EXECUTABLE={}/bin/python".format(python_path),
        "-DPYTHON3_LIBRARY={}/lib/libpython{}m.so".format(python_path, version),
        "-DPYTHON3_PACKAGES_PATH={}/lib/python{}/site-packages/".format(python_path, version),
        "-DPYTHON3_NUMPY_INCLUDE_DIRS={}/lib/python{}/site-packages/numpy/core/include".format(python_path, version),
        "-DPYTHON_INCLUDE_DIR={}/include/python{}m".format(python_path, version),
        _out=sys.stdout,
    )
    sh.make("-j4", _out=sys.stdout)
    sh.make.install(_out=sys.stdout)

def clone(root, uniqname, project):
	into='/tmp/'+root
	mkdir('-p', into)
	url='[email protected]:' + uniqname + '/' + project
	to_path=into + '/' + uniqname

	if 'rerun' in sys.argv:
		if os.path.exists(to_path):
			print('cached {}'.format(to_path))
			repo = git.Repo(to_path)
			return repo, to_path

	print('clone {}/{}'.format(uniqname, project))
	try:
		repo = git.Repo.clone_from(
				url=url,
				to_path=to_path,
				)
		return repo, to_path
	except:
		# Fall back to sh path to grab the error
		try:
			sh.git('clone', url, to_path)
		except sh.ErrorReturnCode as e:
			if 'Connection closed by remote host' in e.stderr.decode('utf8'):
				# gitlab rate-limited us
				time.sleep(3)
				return clone(root, uniqname, project)
			raise

		# Since we're hammering the gitlab server, rarely the first will
		# timeout, but this second chance will succeed. Create a repo object
		# from the path in that case.
		repo = git.Repo(to_path)
		return repo, to_path

    def test_git_dir_from_subdir(self):
        sh.git('init')
        sh.mkdir('foo')
        expected = os.path.join(os.getcwd(), '.git')

        sh.cd('foo')
        self.assertEqual(expected, git_dir())

def split_ann(ann_file):
    if 'tmp' not in ls():
        mkdir('tmp')
    parser = BeautifulSoup(open(ann_file))
    for mistake in parser.find_all('mistake'):
        with open('tmp/%s' % mistake.attrs['nid'], 'a') as f:
            f.write(mistake.__str__())

    def apply(self, config, containers, storage_entry=None):
        # The "entry point" is the way to contact the storage service.
        entry_point = { 'type' : 'titan' }
    
        # List all the containers in the entry point so that
        # clients can connect any of them. 
        entry_point['ip'] = str(containers[0]['data_ip'])

        config_dirs = []
        try:
            for c in containers:
                host_dir = "/tmp/" + self._generate_config_dir(config.uuid, c)
                try:
                    sh.mkdir('-p', host_dir)
                except:
                    sys.stderr.write('could not create config dir ' + host_dir)

                self._apply_rexster(host_dir, storage_entry, c)
                self._apply_titan(host_dir, storage_entry, c)

                # The config dirs specifies what to transfer over. We want to 
                # transfer over specific files into a directory. 
                config_dirs.append([c['container'], 
                                    host_dir + '/*', 
                                    config.config_directory])

        except IOError as err:
            sys.stderr.write('' + str(err))

        return config_dirs, entry_point

    def __init__(self, source, dest, rsync_args='', user=None, **_):
        super().__init__()
        self.user = user
        self.args = (rsync_args, source, dest)

        dest_dir = os.path.split(dest)[0]
        if not os.path.isdir(dest_dir):
            if os.path.exists(dest_dir):
                logger.critical('Destination %s isn\'t valid because a file exists at %s', dest, dest_dir)
                sys.exit(1)
            sh.mkdir('-p', dest_dir)
            if user is not None:
                sh.chown('{}:'.format(user), dest_dir)

        if user is not None:
            self.rsync = sh.sudo.bake('-u', user, 'rsync')
        else:
            self.rsync = sh.rsync
        self.rsync = self.rsync.bake(source, dest, '--no-h', *rsync_args.split(), progress=True)
        self.daemon = True
        self.running = threading.Event()
        self._buffer = ''
        self._status = {
            'running': False,
            'source': source,
            'dest': dest,
        }
        self._status_lock = threading.Lock()

    def compile( self, source_dir, build_dir, install_dir ):
        package_source_dir = os.path.join( source_dir, self.dirname )
        assert( os.path.exists( package_source_dir ) )
        package_build_dir = os.path.join( build_dir, self.dirname )

        sh.cd( os.path.join( package_source_dir, 'scripts/Resources' ) )
        sh.sh( './copyresources.sh' )
        # the install target doesn't copy the stuff that copyresources.sh puts in place
        sh.cp( '-v', os.path.join( package_source_dir, 'bin/Release/Readme.txt' ), os.path.join( install_dir, 'Readme.meshy.txt' ) )
        sh.cp( '-v', '-r', os.path.join( package_source_dir, 'bin/Release_Linux/Resources/' ), install_dir )

        sh.mkdir( '-p', package_build_dir )
        sh.cd( package_build_dir )
        if ( platform.system() == 'Darwin' ):
            sh.cmake(
                '-G', 'Xcode',
                '-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
                '-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'CMake' ),
                package_source_dir,
                _out = sys.stdout )
            sh.xcodebuild( '-configuration', 'Release', _out = sys.stdout )
        else:
            sh.cmake(
                '-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
                '-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'lib/OGRE/cmake' ),
                package_source_dir,
                _out = sys.stdout )
            sh.make( '-j4', 'VERBOSE=1', _out = sys.stdout )
            sh.make.install( _out = sys.stdout )

def create(version_number):
    heading1("Creating new version based on Fedora " + version_number + "\n")

    # Update git and create new version.
    heading2("Updating master branch.")
    print(git.checkout("master"))
    print(git.pull())  # Bring branch current.

    heading2("Creating new branch")
    print(git.checkout("-b" + version_number))  # Create working branch.

    # Get kickstart files.
    heading2("Creating fedora-kickstarts directory\n")
    mkdir("-p", (base_dir + "/fedora-kickstarts/"))
    cd(base_dir + "/fedora-kickstarts/")

    heading2("Downloading Fedora kickstart files.")
    ks_base = "https://pagure.io/fedora-kickstarts/raw/f" \
              + version_number + "/f"

    for file in ks_files:
        file_path = ks_base + "/fedora-" + file

        print ("Downloading " + file_path)
        curl("-O", file_path)

 def compile( self, source_dir, build_dir, install_dir ):
     package_source_dir = os.path.join( source_dir, self.dirname )
     assert( os.path.exists( package_source_dir ) )
     package_build_dir = os.path.join( build_dir, self.dirname )
     runpath_dir = os.path.join( package_source_dir, 'RunPath' )
     if ( not os.path.exists( os.path.join( runpath_dir, 'media.zip' ) ) ):
         sh.cd( runpath_dir )
         sh.wget( '--no-check-certificate', 'https://bitbucket.org/jacmoe/ogitor/downloads/media.zip' )
         sh.unzip( 'media.zip' )
     if ( not os.path.exists( os.path.join( runpath_dir, 'projects.zip' ) ) ):
         sh.cd( runpath_dir )
         sh.wget( '--no-check-certificate', 'https://bitbucket.org/jacmoe/ogitor/downloads/projects.zip' )
         sh.unzip( 'projects.zip' )
     sh.mkdir( '-p', package_build_dir )
     sh.cd( package_build_dir )
     if ( platform.system() == 'Darwin' ):
         sh.cmake(
             '-G', 'Xcode',
             '-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
             '-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'CMake' ),
             package_source_dir,
             _out = sys.stdout )
         sh.xcodebuild( '-configuration', 'Release', _out = sys.stdout )
     else:
         sh.cmake(
             '-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
             '-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'lib/OGRE/cmake' ),
             package_source_dir,
             _out = sys.stdout )
         sh.make( '-j4', 'VERBOSE=1', _out = sys.stdout )
         sh.make.install( _out = sys.stdout )

def generate_template():
    template_file = ""
    if not isdir(build_dir):
        mkdir(build_dir)
    if isdir(build_dir):
        template_file = build_dir + "/dekko.dekkoproject.pot"
        print("TemplateFile: " + template_file)
        cd(build_dir)
        print("Running cmake to generate updated template")
        cmake('..')
        print("Running make")
        make("-j2")
    if isfile(template_file):
        if isdir(po_dir):
            print("Moving template to po dir: " + po_dir)
            mv(template_file, po_dir)
        else:
            print("Couldn't find po dir: " + po_dir)
            cleanup()
            return
    else:
        cleanup()
        print("No template found for: " + template_file)
        return
    print("Cleaning up")
    cleanup()
    print("YeeHaa!")
    print("All done, you need to commit & push this to bitbucket now :-)")
    print("NOTE: this would also be a good time to sync with launchpad, run")
    print("  $ python3 launchpad_sync.py")

def handle_task(task, dotfiles_dir):
    click.echo('handling task: {}'.format(task))

    if 'src' not in task:
        click.echo("you must define at least a 'src' in each task")
        raise click.Abort

    source = os.path.expanduser(task['src'])
    if not os.path.exists(source):
        click.echo('file not found: {}'.format(source))
        raise click.Abort

    _, filename = os.path.split(source)
    subdir = task['subdir'].rstrip('/') if 'subdir' in task else '.'
    target = os.path.abspath(os.path.join(dotfiles_dir, subdir, filename))
    
    # make sure the target directory exists, e.g. .dotfiles/bash/
    target_path, _ = os.path.split(target)
    mkdir('-p', target_path)

    # copy the files
    msg_fmt = 'copying {}: from [{}] to [{}]'
    if os.path.isdir(source):
        click.echo(msg_fmt.format('dir', source, target))
        cp("-r", os.path.join(source, "."), target)
    else:
        click.echo(msg_fmt.format('file', source, target))
        cp(source, target)    

def copy_proguard_mapping(flavor, version_name):
    folder_path = 'releases'
    sh.mkdir("-p", folder_path)
    output_file = '%s/wikipedia-%s.mapping.tar.gz' % (folder_path, version_name)
    input_file = 'wikipedia/build/outputs/mapping/%s/release/mapping.txt' % flavor
    sh.tar('czf', output_file, input_file)
    print ' proguard mapping: %s' % output_file

    def _set_rabbitmq_env(self):

        location = path_expand("~/.cloudmesh/rabbitm")

        if sys.platform == "darwin":
            sh.mkdir("-p", location)
            self.rabbit_env["RABBITMQ_MNESIA_BASE"] = location
            self.rabbit_env["RABBITMQ_LOG_BASE"] = location
            os.environ["RABBITMQ_MNESIA_BASE"] = location
            os.environ["RABBITMQ_LOG_BASE"] = location
            self.rabbit_env["rabbitmq_server"] = \
                "/usr/local/opt/rabbitmq/sbin/rabbitmq-server"
            self.rabbit_env["rabbitmqctl"] = \
                "/usr/local/opt/rabbitmq/sbin/rabbitmqctl"
        elif sys.platform == "linux2":
            sh.mkdir("-p", location)
            self.rabbit_env["RABBITMQ_MNESIA_BASE"] = location
            self.rabbit_env["RABBITMQ_LOG_BASE"] = location
            os.environ["RABBITMQ_MNESIA_BASE"] = location
            os.environ["RABBITMQ_LOG_BASE"] = location
            self.rabbit_env["rabbitmq_server"] = "/usr/sbin/rabbitmq-server"
            self.rabbit_env["rabbitmqctl"] = "/usr/sbin/rabbitmqctl"
        else:
            print("WARNING: cloudmesh rabbitmq user install not supported, "
                  "using system install")

    def __init__(self, test_file, temp_dir, cache=False):
        self.tic = time.time()
        self.test_file = test_file
        self.temp_dir = temp_dir
        self.cache = cache

        self.failed = False
        self.lines = []
        clean_test_file_name = re.sub('^' + re.escape(test_root('data/')), '',
                                      test_file)
        self.say('{}', test_started(clean_test_file_name))
        self.say("Testing {}...", bold(clean_test_file_name))

        self.style = self._get_style()
        sh.mkdir('-p', fail_path(self.style))
        self.style_args = Test._get_style_options(self.style)
        if self.style_args:
            self.say("\tstyling: {}", shell_join(self.style_args))
        self.bib_args = Test._get_bib_options(test_file)
        if self.bib_args:
            self.say("\tbibliography: {}", self.bib_args)
        self.options = self.style_args + self.bib_args

        self.test_name = os.path.join(self.style, os.path.basename(test_file))
        self.test_out = os.path.join(self.temp_dir, self.test_name)
        self.test_err = self.test_out + '.err'
        _, ext = os.path.splitext(test_file)
        self.test_new = self.test_out + '.new.' + ext

def git_clone_to_local(dest_directory, webhook_data):
    git = sh.git.bake()
    logger.debug('Making destination directory %s' % dest_directory)
    print ('Making destination directory %s' % dest_directory)
    sh.mkdir('-p', dest_directory)
    sh.cd(dest_directory)
    logger.debug("checking for repo_name %s in %s" % (webhook_data.repo_name, sh.pwd()))
    if not os.path.exists(webhook_data.repo_name):
        logger.debug("Cloning new repository")
        print(git.clone(webhook_data.repo_url, webhook_data.repo_name))
    sh.cd(webhook_data.repo_name)
    print(git.fetch('--all'))

    try:
        git('show-ref', '--heads', webhook_data.branch_name)
        branch_exists = True
    except:
        branch_exists = False

    if branch_exists is False and not webhook_data.is_tag():
        print(git.checkout('-b', webhook_data.branch_name,
                           'origin/%s' % webhook_data.branch_name))
    elif branch_exists:
        git.checkout(webhook_data.branch_name)

    print(git.pull())
    print webhook_data.before, webhook_data.after

    def update_cache(self):
        if self.repo_url == '':
            print("repo_url is not set, cannot update")
            return

        if not self.test_cache():
            self.purge_cache()
            mkdir(os.path.join(self.cache_dir, 'repodata'))

            index_file_url = '/'.join([self.repo_url, self.index_file])
            index_file_path = os.path.join(self.cache_dir, self.index_file)

            try:
                print("Downloading index file '{0}' --> '{1}' ...".format(
                    index_file_url, index_file_path
                ))
                urlretrieve(index_file_url, index_file_path)
            except:
                self.broken = True
                return

            try:
                xmlroot = etree.parse(index_file_path).getroot()
                xmlns = xmlroot.nsmap[None]
                for item in xmlroot.findall("{{{0}}}data".format(xmlns)):
                    for subitem in item.findall("{{{0}}}location".format(xmlns)):
                        location = subitem.get('href')
                        url = '/'.join([self.repo_url, location])
                        path = '/'.join([self.cache_dir, location])
                        print("Downloading file '{0}' --> '{1}' ...".format(
                            url, path
                        ))
                        urlretrieve(url, path)
            except:
                self.broken = True