def test_repository_add_search_keyword_success(self):
        repo = Repository(repo_name)
        doc = test_doc(repo)

        result = repo.search_keywords('jungle')

        self.assertEqual(len(result), 1)

 def cmd_checkout(branch):
     b = Branch()
     b.switch_branch(branch)
     repo = Repository()
     pre_entries = dict(repo.index.entries)  
     repo.rebuild_index_from_commit(repo.branch.head_commit)
     repo.rebuild_working_tree(pre_entries)

    def __init__(self, filename=None):
        if not filename:
            raise FileRepositoryException("Please set a filename")

        self._filename = filename
        Repository.__init__(self)
        self._load()

    def test_repository_add_remove_has_not(self):
        repo = Repository(repo_name)
        doc = test_doc(repo)

        repo.remove_document(doc)

        self.assertFalse(repo.has_document_uuid(doc.uuid))

def app_setup(app):
    app.config['CELERY_ACCEPT_CONTENT'] = ['json']
    app.config['CELERY_TASK_SERIALIZER'] = 'json'
    app.config['CELERY_RESULT_SERIALIZER'] = 'json'
    app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/2'
    app.config['CELERY_BACKEND'] = 'redis://localhost:6379/3'
    app.config['CELERY_QUEUES'] = (
        Queue('transplant', Exchange('transplant'), routing_key='transplant'),
    )

    app.src_dir = tempfile.mkdtemp(dir=test_temp_dir)
    app.dst_dir = tempfile.mkdtemp(dir=test_temp_dir)
    app.config['TRANSPLANT_WORKDIR'] = tempfile.mkdtemp(dir=test_temp_dir)
    app.config['TRANSPLANT_REPOSITORIES'] = [{
        'name': 'test-src',
        'path': app.src_dir
    }, {
        'name': 'test-dst',
        'path': app.dst_dir
    }]

    app.src = Repository.init(app.src_dir)
    app.dst = Repository.init(app.dst_dir)

    _set_test_file_content(app.src_dir, "Hello World!\n")
    app.src.commit("Initial commit", addremove=True, user="Test User")
    app.dst.pull(app.src_dir, update=True)

class BaseHaskiAction(object):
    """This class is the abstract base class for Haski tasks.
    """
    __metaclass__ = ABCMeta

    def __init__(self):
        self._repository = Repository()

    @abstractmethod
    def __call__(self, namespace):
        """This method is invoked by argparse and should do the actuall work.
        By default, it is abstract and does nothing, so it forces you to
        implement it.
        """
        pass

    def get_commit(self, namespace):
        """This function gets the commit wanted commit on which an action is to
        be performed.
        """
        commit = self._repository.get_commit(namespace.revision)
        return commit

    def get_repository_location(self):
        """Returns the directory where this repository is located.
        """
        return self._repository.get_path()

 def test_status_untracked_files(self):
     path, content = ('1.txt', '1\n')
     write_to_file(path, content)
     repo = Repository()
     untracked_files = repo.get_untracked_files()
     self.assertEqual(untracked_files, ['1.txt'])
     Command.cmd_status()

    def validate_name(self, min_length, model, parameter):
        if len(parameter) < min_length:
            message = "The name should be at least {} symbols"
            raise Exception(message.format(min_length))

        repository = Repository(self.__db_name)
        if repository.is_name_used(model, parameter):
            raise Exception("This name is already taken")

 def test_reset_default(self):
     Command.cmd_reset(self.first_commit, is_soft=False, is_hard=False)
     self.assertEqual(Branch().head_commit, self.first_commit)
     repo = Repository()
     uncommitted_files = repo.get_uncommitted_files()
     unstaged_files = repo.get_unstaged_files()
     self.assertFalse(uncommitted_files['modified'])
     self.assertIn(self.path, unstaged_files['modified'])

class TwitterDataProcess():
    def __init__(self, topics, file_name):
        self.file_name = file_name
        self.topics = topics
        self.tweets = []
        self.repository = Repository()
        self.process_tweets()

    @staticmethod
    def compact_tweet_text(tweet_text):
        return tweet_text.replace('\n', ' ').replace('\r', '').lower()

    @staticmethod
    def to_dictionary(coordinate):
        keys = ["Lat", "Lng"]
        return dict(zip(keys, coordinate))

    def check_key_words(self, topic_key, tweet_text):
        if re.search(topic_key, tweet_text):
            for word in self.topics.get(topic_key):
                if re.search(word, tweet_text):
                    return True
        return False

    def process_tweets(self):
        self.tweets = []
        with open(self.file_name, "r") as tweets_file:
            for line in tweets_file:
                tweet = dict()
                tweet_valid = False
                try:
                    if line.strip() != '':
                        raw_tweet = json.loads(line)
                        text = TwitterDataProcess.compact_tweet_text(raw_tweet['text'])
                        for topic_key in self.topics:
                            if self.check_key_words(topic_key, text):
                                tweet_valid = True
                                tweet[topic_key] = True
                            else:
                                tweet[topic_key] = False
                        if tweet_valid:
                            tweet['text'] = text
                            tweet['lang'] = raw_tweet['lang']
                            tweet['city'] = raw_tweet['place']['name'] if raw_tweet['place'] is not None else None
                            if raw_tweet['geo'] is None:
                                tweet['coordinates'] = False
                            else:
                                tweet['coordinates'] = True
                                tweet.update(TwitterDataProcess.to_dictionary(raw_tweet['geo']['coordinates']))
                                self.tweets.append(tweet)
                            if len(self.tweets) > 1000:
                                self.repository.save_many(self.tweets)
                                self.tweets = []

                except Exception as e:
                    print(str(e))
                    continue
        self.repository.save_many(self.tweets)

def add_endpoint(my_request):

    if not my_request.pmh_url:
        return None

    endpoint_with_this_id = Endpoint.query.filter(Endpoint.repo_request_id==my_request.id).first()
    if endpoint_with_this_id:
        print u"one already matches {}".format(my_request.id)
        return None

    raw_endpoint = my_request.pmh_url
    clean_endpoint = raw_endpoint.strip()
    clean_endpoint = clean_endpoint.strip("?")
    clean_endpoint = re.sub(u"\?verb=.*$", "", clean_endpoint, re.IGNORECASE)
    print u"raw endpoint is {}, clean endpoint is {}".format(raw_endpoint, clean_endpoint)

    matching_endpoint = Endpoint()
    matching_endpoint.pmh_url = clean_endpoint

    repo_matches = my_request.matching_repositories()
    if repo_matches:
        matching_repo = repo_matches[0]
        print u"yay! for {} {} matches repository {}".format(
            my_request.institution_name, my_request.repo_name, matching_repo)
    else:
        print u"no matching repository for {}: {}".format(
            my_request.institution_name, my_request.repo_name)
        matching_repo = Repository()

    # overwrite stuff with request
    matching_repo.institution_name = my_request.institution_name
    matching_repo.repository_name = my_request.repo_name
    matching_repo.home_page = my_request.repo_home_page
    matching_endpoint.repo_unique_id = matching_repo.id
    matching_endpoint.email = my_request.email
    matching_endpoint.repo_request_id = my_request.id
    matching_endpoint.ready_to_run = True
    matching_endpoint.set_identify_and_initial_query()

    db.session.merge(matching_endpoint)
    db.session.merge(matching_repo)
    print u"added {} {}".format(matching_endpoint, matching_repo)
    print u"see at url http://unpaywall.org/sources/repository/{}".format(matching_endpoint.id)
    safe_commit(db)
    print "saved"

    print "now sending email"
    # get the endpoint again, so it gets with all the meta info etc
    matching_endpoint = Endpoint.query.get(matching_endpoint.id)
    matching_endpoint.contacted_text = "automated welcome email"
    matching_endpoint.contacted = datetime.datetime.utcnow().isoformat()
    safe_commit(db)
    send_announcement_email(matching_endpoint)

    print "email sent"

    return matching_endpoint

 def __init__(self, repoid, repoConfig, maxFileLength=1024*1024*1024*2,
              retryTime=3, socketTimeout=5):
     Repository.__init__(self, repoid)
     self.repoConfig = repoConfig
     self.cachedir = None
     self.expireTime = None
     self.enabled = bool(repoConfig.enabled)
     self.maxFileLength = maxFileLength
     self.retryTime = retryTime
     self.socketTimeout = socketTimeout

def main(argv):

    config_file = None
    read_only = False
    verbose = False

    try:
        # -h (optional), -c (mandatory, hence 'c:'), -r (optional)
        opts, args = getopt.getopt(argv,"hc:rv",[])
    except getopt.GetoptError:
        print 'repository.py [-h] [-r] [-v] -c <cfgfile>'
        sys.exit(2)

    for opt, arg in opts:
        if opt == '-h':
            print 'repository.py [-h] [-r] [-v] -c <cfgfile>'
            sys.exit(0)
        elif opt in ("-c", "--cfgfile"):
            config_file = arg
        elif opt == '-r':
            read_only = True
        elif opt == '-v':
            verbose = True

    if config_file is None:
        print 'repository.py [-h] [-r] [-v] -c <cfgfile>'
        sys.exit(2)

    if verbose: print 'Using cfg file:', config_file
    repository = Repository(config_file, verbose)
    
    config = ConfigParser.RawConfigParser()
    config.read(config_file)

    sensor_names = config.get("sensors", "names").strip().split(',')
    if verbose: print 'Sensor names:', sensor_names

    sensors = []

    for name in sensor_names:
        if verbose: print 'Adding sensor:', name
        sensors.append(Sensor(name, config_file))
    
    for sensor in sensors:
        if verbose: print 'Reading sensor', sensor.sensor_id

        readings = sensor.get_readings()
        if readings is not None:
            if read_only:
                s=json.dumps(readings, sort_keys=True, indent=4, separators=(',', ': '))
                print s
            else:
                repository.save_readings(readings)

    print 'Done'

	def repositories(self):
		_repos = self.db.query("SELECT id FROM repositories WHERE distro_id = %s", self.id)

		repos = []
		for repo in _repos:
			repo = Repository(self.pakfire, repo.id)
			repo._distro = self

			repos.append(repo)

		return sorted(repos)

	def get_repo(self, name):
		repo = self.db.get("SELECT id FROM repositories WHERE distro_id = %s AND name = %s",
			self.id, name)

		if not repo:
			return

		repo = Repository(self.pakfire, repo.id)
		repo._distro = self

		return repo

    def prepare_mock_repositories(self):
        self.src_dir = tempfile.mkdtemp()
        self.dst_dir = tempfile.mkdtemp()
        self.workdir = tempfile.mkdtemp()

        self.src = Repository.init(self.src_dir)
        self.dst = Repository.init(self.dst_dir)

        self._set_test_file_content(self.src_dir, "Hello World!\n")
        self.src.commit("Initial commit", addremove=True)
        self.dst.pull(self.src_dir, update=True)

    def transplant(self, src_dir, dst_dir, items):
        src_repo = Repository(src_dir)
        dst_repo = Repository(dst_dir)

        try:
            for item in items:
                self._transplant_item(src_repo, dst_repo, item)

            tip = dst_repo.id(id=True)
            logger.info('tip: %s', tip)
            return {'tip': tip}

        finally:
            self._cleanup(dst_repo)

 def test_status_unstaged_files(self):
     file_list = [('1.txt', '1\n'), ('2.txt', '2\n')]
     for path, content in file_list:
         write_to_file(path, content)
         Command.cmd_add(path)
     
     write_to_file(file_list[0][0], '11\n')
     os.remove(file_list[1][0])
     
     repo = Repository()
     unstaged_files = repo.get_unstaged_files()
     
     self.assertEqual(unstaged_files['modified'], [file_list[0][0]])
     self.assertEqual(unstaged_files['deleted'], [file_list[1][0]])
     Command.cmd_status()

def getLocally():

    logger = logging.getLogger(__name__)
    logger.setLevel(logging.WARNING)
    file_handler = logging.FileHandler('mergehistory.log')
    logger.addHandler(file_handler)
    session = getSession()
    line_number = 0
    with gzip.open(zipfile,"rb") as input_file:

        for line in input_file:
            line_number += 1
            try:
                user_repositories = json.loads(line)

                for repo in user_repositories:
                    if "created_at" in repo and repo["created_at"]:
                        year = repo["created_at"][:4]
                        is_success = Repository.update(session, repo['id'], creation_date = year, main_lang = repo['language'])

                        if is_success:
                            logger.info('%d %s %s' %(repo['id'] , year , repo['language']))
                        else:
                            logger.warning(str(line_number) + ':' + str(repo['id']) +' update was unsuccessful')

                    else:
                        print repo
                logger.info('user repos updated')
                session.commit()
                file_handler.flush()
            except:
                logger.error(str(line_number) + ':' + line)
                
    logging.shutdown()

 def print_repos(prefix):
     paths = os.listdir(os.path.join(config.REPO_ROOT, prefix))
     print "\n[ Repositories in %s ]" % os.path.join(config.REPO_ROOT, prefix)
     paths.sort()
     for path in paths:
         if Repository.exists(os.path.join(config.REPO_ROOT, prefix, path)):
             print path