def update_dataset_lists_finalize(self):
        
        concepts = {}
        codelists = {}
        dimension_keys = []
        attribute_keys = []
        
        for key, value in self.dataset.concepts.items():
            key_slug = slugify(key, save_order=True)
            concepts[key_slug] = value
            
        for key, value in self.dataset.codelists.items():
            new_value = {}
            for k, v in value.items():
                new_value[slugify(k, save_order=True)] = v
            codelists[slugify(key, save_order=True)] = new_value
            
        for key in self.dataset.dimension_keys:
            dimension_keys.append(slugify(key, save_order=True))

        if self.dataset.attribute_keys:
            for key in self.dataset.attribute_keys:
                attribute_keys.append(slugify(key, save_order=True))
            
        self.dataset.concepts = concepts
        self.dataset.codelists = codelists
        self.dataset.dimension_keys = dimension_keys

        if self.dataset.attribute_keys:
            self.dataset.attribute_keys = attribute_keys

    def __init__(self, data = None):
        if data == None:
            return
        self.label = data.get('label')
        if (data.get('name', None)):
            self.name = slugify(str(data.get('name')), max_length=30, separator="_")
        else:
            self.name = slugify(str(data.get('label')), max_length=30, separator="_")

        #check if name is already taken
        if Dataset.by_name(self.name):
            for x in range(10):
                newname = self.name + "_" + str(x)
                if not Dataset.by_name(newname):
                    self.name = newname
                    break


        self.description = data.get('description')
        self.ORoperations = data.get('ORoperations', {})
        self.mapping = data.get('mapping', {})
        self.prefuncs = data.get('prefuncs', {})
        self.created_at = datetime.utcnow()
        self.dataType = data.get('dataType')
        if type(data.get('dataorg')) == int:
            self.dataorg = DataOrg.by_id(data.get('dataorg'))
        else:
            try:
                self.dataorg = data.get('dataorg')
            except Exception, e:
                print "failed to load the dataorg for dataset"
                print e

 def save(self, *args, **kwargs):
     if not self.pk:
         slug_entry = self.name
         chk = Category.objects.filter(slug=slugify(slug_entry))
         if len(chk): slug_entry = slug_entry + "-" + str(len(chk))
         self.slug = slugify(slug_entry)
     super(Category, self).save(*args, **kwargs)

    def crea_group_with_manager(name_user, name_group):
        # Recuperation de la surcharge de user
        User = get_user_model()

        # Test si nom user est deja prit
        lUser = list(User.objects.all())
        for u in lUser:
            if u.name_long == name_user:
                raise Exception('That user name already exists')

        lGroup = list(GroupProfile.objects.all())
        for g in lGroup:
            if g.title == name_group or g.slug == slugify(name_group):
                raise('That group name already exists')

        user = User.objects.create_user(name_user, None, name_user)
        user.save()

        group = GroupProfile()
        group.title = name_group
        group.slug = slugify(name_group)
        group.description = name_group

        group.save()

        group.join(user, role="manager")
        group.save()

        return True

 def download_ticket(self, ticket, destination_dir='.', override=False):
     route = '%s_%s' % (slugify(ticket.route_from, separator='_')[:3],
                        slugify(ticket.route_to, separator='_')[:3])
     filename = '%s/%s_%s_%s.pdf' % (
         destination_dir, ticket.date.strftime('%Y%m%d%H%M'),
         route, ticket.number)
     if not os.path.exists(filename) or override:
         if not os.path.exists(destination_dir):
             os.mkdir(destination_dir)
         response = self.session.get(
             self.PDF_URL % ticket.number[3:], stream=True)
         total = int(response.headers.get('Content-Length', 0))
         if not total:
             return
         name = os.path.split(filename)[1]
         chunk_size = 1024
         progress = tqdm(
             total=total, leave=True, unit_scale=chunk_size, unit='B',
             desc='Downloading %s' % (name,))
         with open(filename, 'wb') as file_handler:
             for chunk in response.iter_content(chunk_size=chunk_size):
                 if chunk:
                     file_handler.write(chunk)
                     progress.update(chunk_size)
         progress.close()
     return open(filename)

def main():

    is_rewrite = 'rewrite' in sys.argv
    cards = json.load(open(CARDS_FILE_PATH))

    print("From '{}' loaded {} cards".format(CARDS_FILE_PATH, len(cards)))

    if not os.path.isdir('cards'):
        os.mkdir('cards')

    added_cards_count = 0

    for card in cards:
        cycle_number = card['cyclenumber']

        if cycle_number >= 1:  # ignore alternates and specials
            set_name = card['setname']
            set_name = '{}-{}'.format(SET_ORDER.get(set_name, ''), set_name)

            dir_name = '{:02}-{}'.format(cycle_number, slugify(set_name))
            dir_path = os.path.join('cards', dir_name)

            file_name = '{:03}-{}.yaml'.format(card['number'], slugify(card['title']))
            file_path = os.path.join(dir_path, file_name)

            if not os.path.isdir(dir_path):
                os.mkdir(dir_path)

            # Add absent cards only
            if is_rewrite or not os.path.isfile(file_path):

                card_file = open(file_path, 'w')

                card_yaml = OrderedDict()

                card_yaml['side'] = card['side']
                card_yaml['faction'] = card['faction']
                card_yaml['type'] = card['type']
                card_yaml['uniqueness'] = card['uniqueness']

                card_yaml['obvious'] = False
                card_yaml['progress'] = 0.0

                card_yaml['title'] = card['title']
                card_yaml['title_ru'] = 'нет'

                text = clean_breaks(card['text'])
                card_yaml['text'] = folded(text)
                card_yaml['text_ru'] = folded(text)

                if 'flavor' in card:
                    flavor = clean_breaks(card['flavor'])
                    card_yaml['flavor'] = folded(flavor)
                    card_yaml['flavor_ru'] = folded(flavor)

                yaml.dump(card_yaml, card_file, default_flow_style=False, allow_unicode=True, indent=4, width=70)

                added_cards_count += 1

    print('Added {} cards'.format(added_cards_count))

def ensure_unify_datasets_exist():
    """
    Read the unify datasets to create from the CSV file.

    1. Check if they exist.
    2. If they don't, create 'em.
    3. There is no step 3.
    4. Profit
    """
    dc.ensure_publisher('unify')
    unifyfile = DATA_DIR/'datasets.csv'
    with unifyfile.csv(header=True) as csv:
        for row in csv:
            if row.source == 'UNIFY2':
                dc.Dataset.create_or_update(
                    name=slugify(row.title).lower(),
                    title=row.title,
                    state='active',
                    private=row.public=='N', 
                    license_id='ogl',
                    url='http://data.england.nhs.uk',
                    owner_org='unify',
                    resources=[]
                )
                print slugify(row.title).lower()
    return

    def build_docs(self, row):

        # Clean expense string so that is is numerical (e.g. turn blank string to 0).
        cost = row[9].replace(',', '')
        if not cost.strip():
            cost = 0

        # Create doc.
        doc = {
            'region': {
                'name': self.get_region(),
                'slug': slugify(self.get_region(), to_lower=True),
                'subregion':{
                    'name': cyrtranslit.to_latin(row[0]),
                    'slug': cyrtranslit.to_latin(slugify(row[0], to_lower=True)),
                }
            },
            'activity':{
                'id': int(row[1]),
                'description': cyrtranslit.to_latin(row[2])
            },
            'dataset': {
                'name': self.get_dataset(),
                'slug': slugify(self.get_dataset(), to_lower=True)
            },
            'cost': cost,
            'year': 2010
        }

        # Console output to provide user with feedback on status of importing process.
        print '%s - %s: %s (%s %i)' % (doc['activity']['id'], doc['activity']['description'], doc['cost'], doc['region']['name'], doc['year'])

        return [doc]

def prepare_folder(list):
    global type
    type = list['extractor_key']
    if "www.youtube.com/user/" in sys.argv[1]:
        type = "user"
    global title
    global title_html

    if type == "YoutubePlaylist":
        title = slugify.slugify(list['title'])
        title_html = list['title']
    else:
        title =  slugify.slugify(list.get('entries')[0].get('uploader'))
        title_html = list.get('entries')[0].get('uploader')



    global scraper_dir
    scraper_dir = script_dirname + "build/" + title + "/"

    if not os.path.exists(scraper_dir):
        os.makedirs(scraper_dir)
    if not os.path.exists(scraper_dir+"CSS/"):
        shutil.copytree("templates/CSS/", scraper_dir+"CSS/")
    if not os.path.exists(scraper_dir+"JS/"):
        shutil.copytree("templates/JS/", scraper_dir+"JS/")
    get_user_pictures(list.get('entries')[0].get('uploader_id'))

    global color
    color = colorz(scraper_dir+"CSS/img/header.png", 1)[0];

    global background_color
    background_color = solarize_color(color);

    def default_content(cls):
        ret = {}

        title = 'About text (left column)'
        slug = slugify(title, to_lower=True)
        ret[slug] = cls(title=title, slug=slug, content=(
            '<p>The aim of this app is to demonstrate that, with the '
            'help of modern JS libraries, and with some well-'
            'thought-out server-side snippets, it\'s now perfectly '
            'possible to "bake in" live in-place editing for '
            'virtually every content element in a typical '
            'brochureware site.</p>'),
            active=True)

        title = 'About text (right column)'
        slug = slugify(title, to_lower=True)
        ret[slug] = cls(title=title, slug=slug, content=(
            '<p>This app is not a CMS. On the contrary, think of it '
            'as a proof-of-concept alternative to a CMS. An '
            'alternative where there\'s no "admin area", there\'s '
            'no "editing mode", and there\'s no "preview '
            'button".</p>'),
            active=True)

        title = 'About text (below columns)'
        slug = slugify(title, to_lower=True)
        ret[slug] = cls(
            title=title, slug=slug,
            content="<p>There's only direct manipulation.</p>",
            active=True)

        return ret

def get_station_info(station_code, station_data_dict):
    station = {
        'kodi': station_code,
        'emri': station_data_dict[station_code]['name'],
        'slug': slugify(station_data_dict[station_code]['name']),
        'kordinatat': {
            'gjatesi': float(station_data_dict[station_code]['longitude']),
            'gjeresi': float(station_data_dict[station_code]['latitude'])
        },
        'gjiriLumit': {
            'emri': station_data_dict[station_code]['riverBasin'],
            'slug': slugify(station_data_dict[station_code]['riverBasin'])
        },
        'lumi': {
            'emri': station_data_dict[station_code]['river'],
            'slug': slugify(station_data_dict[station_code]['river'])
        },
        'regjioniDetit': {
            'emri': station_data_dict[station_code]['seaRegion'],
            'slug': slugify(station_data_dict[station_code]['seaRegion'])
        },
        'vendMostrimi': float(station_data_dict[station_code]['catchmentArea']),
        'dendesiaPopullates': float(station_data_dict[station_code]['populationDensity']),
        'lartesia': int(station_data_dict[station_code]['altitude'])
    }

    return station

    def search_page(self, response):
        trs = response.xpath('.//table/tbody/tr')
        for tr in trs:
            tds = [x.extract() for x in tr.xpath('./td/text()')]
            scheme = tds[3].strip()
            if 'Sum total' in scheme:
                continue
            amount = float(tds[4].replace('.', '').replace(',', '.'))
            recipient_name = tds[0]
            if self.NUM_ONLY_RE.match(recipient_name) is not None:
                recipient_id = u'SI-%s-%s' % (self.YEAR, recipient_name)
                recipient_name = ''
            else:
                recipient_id = u'SI-%s-%s' % (slugify(tds[1]), slugify(recipient_name))

            recipient_location = u'%s, %s' % (tds[1], tds[2])
            yield FarmSubsidyItem(
                year=self.YEAR,
                scheme=scheme,
                amount=amount,
                recipient_id=recipient_id,
                recipient_name=recipient_name,
                recipient_location=recipient_location,
                country='SI', currency='EUR',
            )

def new():
  """Create a new post or page."""

  date = datetime.now()
  page_type = click.prompt('Create new post/page', type=click.Choice(['post',
                           'page']), default='post')
  page_attributes['title'] = click.prompt('Title', default='New ' + page_type)
  page_attributes['date'] = date.strftime(config['internal_date_format'])
  page_attributes['template'] = config[page_type + '_template']
  if page_type == 'post':

    # i.e. "2014-05-10-post-title.md"

    file_name = date.strftime(config['post_prefix_format']) \
      + slugify(page_attributes['title']) + '.md'
  else:

    # i.e. "page-title.md"

    file_name = slugify(page_attributes['title']) + '.md'
  file_path = os.path.join(config['src_dir'], file_name)
  if os.path.isfile(file_path):
    click.echo('A file with the same name already exists.')
  else:
    with codecs.open(file_path, 'w', encoding='utf-8') as f:
      f.write(config['delimiter'])
      f.write(yaml.dump(page_attributes, default_flow_style=False))
      f.write(config['delimiter'])
      f.write('\n')

def create_new():
    app.logger.debug("Form keys: " + ", ".join(request.form.keys()))
    if request.method == "GET":
        return render_template("new.html",
                               user=slugify(request.args.get("user", "")),
                               key=slugify(request.args.get("key", "")),
                               code=request.args.get("code", ""))
    else:
        user = slugify(request.form.get("user", "Anonymous"))
        try:
            key = slugify(request.form.get("key", modelo.gen_key(user)))
        except KeyError:
            app.logger.error("Too many retries to generate a key")
            abort(500)
        code = request.form.get("code", "").strip()

        if (code is None or len(code) == 0):
            flash("No code to submit?")
            return redirect(url_for("create_new", user=user, key=key))
        elif modelo.is_used_key(user, key):
            flash("Select another key, that one has already been taken!")
            return redirect(url_for("create_new", user=user,
                                    key=key, code=code))
        else:
            modelo.add_pasta(user, key, code)
            return redirect(url_for("get_pasta", user=user, key=key))

    def get_slug_default(self):
        """
        Naively constructs a translated default slug from the object. For
        better results, just set the `slug_default` property on the class to a
        lazy translated string. Alternatively, override this method if you need
        to more programmatically determine the default slug.

        Example: If your model is "news article" and your source field is
        "title" this will return "news-article-without-title".
        """
        if self.slug_default:
            # Implementing class provides its own, translated string, use it.
            return force_text(self.slug_default)

        object_name = self._meta.verbose_name

        # Introspect the field name
        try:
            trans_meta = self.translations.model._meta
            source_field = trans_meta.get_field(
                self.slug_source_field_name)
            field_name = getattr(source_field, 'verbose_name')
        except Exception:
            field_name = _('name')

        slug_default = _("{0}-without-{1}").format(
            slugify(force_text(object_name)),
            slugify(force_text(field_name)),
        )
        return slug_default

def udemy_dl(username, password, course_link, lecture_start=1, lecture_end=None, save_links=False, safe_file_names=False, just_list=False, dest=""):
    # pylint: disable=too-many-arguments
    """Login into udemy and do all magic."""
    login(username, password)

    course_id = get_course_id(course_link)

    check_course_status(course_id)

    last_chapter = -1

    for data in get_data_links(course_id, lecture_start, lecture_end):
        if save_links:
            save_link(data['data_url'], dest, data['data_type'])
        else:
            try:
                directory = '{0:02d} {1!s}'.format(data['chapter_number'], safeencode(data['chapter']))

                if safe_file_names:
                    directory = slugify(directory, lower=True, spaces=False, ok='.', only_ascii=True)
                else:
                    directory = sanitize_path(directory)

            except AttributeError:
                # Fix for untitled opening chapter

                if safe_file_names:
                    directory = '00-opening'
                else:
                    directory = '00 Opening'

            if dest:
                directory = os.path.join(dest, directory)

            filename = '{0:03d} {1!s}'.format(data['lecture_number'], safeencode(data['lecture']))

            if safe_file_names:
                filename = slugify(filename, lower=True, spaces=False, ok='.', only_ascii=True)
            else:
                filename = sanitize_path(filename)

            if just_list:
                if last_chapter != data['chapter_number']:
                    last_chapter = data['chapter_number']
                    print('\r\n{0:02d} {1!s}\r\n=========================='.format(last_chapter, safeencode(data['chapter'])))
                print('{0:03d} {1!s}'.format(data['lecture_number'], safeencode(data['lecture'])))
            else:
                data_url = data['data_url']
                data_type = data['data_type']
                attached_info = data['attached_info']
                caption_list = data['caption_list']
                get_data(directory, filename, data_url, data_type, attached_info, caption_list)

    if os.path.exists(dest) and save_links:
        print('Links successfully saved to : {0!s}\n'.format((os.path.abspath(dest))))

    print('Logging out...', end=' ')
    sys.stdout.flush()
    session.get('http://www.udemy.com/user/logout')
    print('Done')

    def template_to_path(self, track: dict) -> str:
        """Create valid filepath based on template

        :param track: track metadata
        :return: filepath
        """
        logging.debug(" Generating filepath/trackname..")
        path = self.template

        if self.no_slugify:
            path = path.replace("%{artist}", track['artist'])
            path = path.replace("%{album}", track['album'])
            path = path.replace("%{title}", track['title'])
        else:
            path = path.replace("%{artist}", slugify(track['artist']))
            path = path.replace("%{album}", slugify(track['album']))
            path = path.replace("%{title}", slugify(track['title']))

        if track['track'] == "None":
            path = path.replace("%{track}", "Single")
        else:
            path = path.replace("%{track}", str(track['track']).zfill(2))

        path = u"{0}/{1}.{2}".format(self.directory, path, "mp3")

        logging.debug(" filepath/trackname generated..")
        logging.debug("\n\tPath: {}".format(path))
        return path

    def _add_family_info(self, project_id, family_id, individuals):
        """
        Add all the background info about this family
        We try to keep this as simple as possible - just IDs
        After this is run, variants are ready to be loaded
        """

        if self.family_exists(project_id, family_id):
            #raise Exception("Family (%s, %s) already exists" % (project_id, family_id))
            return

        for indiv_id in individuals:
            if not self.individual_exists(project_id, indiv_id):
                self.add_individual(project_id, indiv_id)

        family_coll_name = "family_%s_%s" % (slugify.slugify(project_id, separator='_'),
                                             slugify.slugify(family_id, separator='_'))
        family = {
            'project_id': project_id,
            'family_id': family_id,
            'individuals': individuals,
            'coll_name': family_coll_name,
            'status': 'loading'
        }

        family_collection = self._db[family_coll_name]
        self._index_family_collection(family_collection)

        self._db.families.save(family, safe=True)

 def create_dbdict(self):
     print('Creating dbdict for cultivar "{}"...'.format(self.h3))
     self._dbdict['name'] = first_text(self.h3)
     self._dbdict['subtitle'] = first_text(self.subtitle_em)
     self._dbdict['botanical_names'] = first_text(self.bn_em)
     self._dbdict['description'] = tags_to_str(self.ps)
     self._dbdict['new_for'] = '2017' if self.new_for else ''
     self._dbdict['favorite'] = True if self.favorite else False
     self._dbdict['images'] = [
         i['src'].replace('\n', '') for i in self.images
     ]
     self._dbdict['packets'] = self.get_packet_dicts()
     self._dbdict['veg_info'] = {}
     if self.veg_em:
         abbrs = self.veg_em.find_all('abbr')
         self._dbdict['veg_info']['open_pollinated'] = False
         for abbr in abbrs:
             abbr.extract()
             if '(OP)' in abbr.text:
                 self._dbdict['veg_info']['open_pollinated'] = True
         self._dbdict['veg_info']['maturation'] = str_contents(self.veg_em)
     try:
         self._dbdict['slug'] = slugify(self.tag['id'])
     except KeyError:
         try:
             self._dbdict['slug'] = slugify(self.h3['id'])
         except KeyError:
             try:
                 self._dbdict['slug'] = slugify(self.tag.img['id'])
             except KeyError:
                 self._dbdict['slug'] = None

 def test_fold_abbr_4(self):
     slugify = Slugify(fold_abbrs=True)
     self.assertEqual('mind-in-a-box', slugify('mind.in.a.box'))
     self.assertEqual('mind-in-a-b-c-box', slugify('mind.in.a.b.c.box'))
     self.assertEqual('a-b-c-box', slugify('.a.b.c.box'))
     self.assertEqual('abcbox', slugify('a.b.c.box'))
     self.assertEqual('abcb-ox', slugify('a.b.c.b ox'))