def init_app(app):
    if not app.config['LEGAL_ARCHIVE']:
        return

    endpoint_name = LEGAL_ARCHIVE_NAME
    service = LegalArchiveService(endpoint_name, backend=get_backend())
    LegalArchiveResource(endpoint_name, app=app, service=service)

    endpoint_name = LEGAL_ARCHIVE_VERSIONS_NAME
    service = LegalArchiveVersionsService(endpoint_name, backend=get_backend())
    LegalArchiveVersionsResource(endpoint_name, app=app, service=service)

    endpoint_name = LEGAL_ARCHIVE_HISTORY_NAME
    service = LegalArchiveHistoryService(endpoint_name, backend=get_backend())
    LegalArchiveHistoryResource(endpoint_name, app=app, service=service)

    endpoint_name = LEGAL_PUBLISH_QUEUE_NAME
    service = LegalPublishQueueService(endpoint_name, backend=get_backend())
    LegalPublishQueueResource(endpoint_name, app=app, service=service)

    privilege(name=LEGAL_ARCHIVE_NAME, label='Legal Archive', description='Read from legal archive')

    superdesk.command('legal_publish_queue:import', ImportLegalPublishQueueCommand())
    superdesk.command('legal_archive:import', ImportLegalArchiveCommand())

def process_validators(filepath):
    """
    This function upserts the validators into the validators collections.
    The format of the file used is JSON.
    :param filepath: absolute filepath
    :return: nothing
    """
    if not os.path.exists(filepath):
        raise FileNotFoundError

    with open(filepath, 'rt') as validators:
        json_data = json.loads(validators.read())
        populate_validators(json_data)


class ValidatorsPopulateCommand(superdesk.Command):
    """
    Class defining the populate validators command.
    """
    option_list = (
        superdesk.Option('--filepath', '-f', dest='filepath', required=True),
    )

    def run(self, filepath):
        process_validators(filepath)


superdesk.command('validators:populate', ValidatorsPopulateCommand())

    public_methods = ['POST']


class PrepopulateService(BaseService):
    def create(self, docs, **kwargs):
        for doc in docs:
            if doc.get('remove_first'):
                drop_elastic(superdesk.app)
                drop_mongo(superdesk.app)
            user = get_resource_service('users').find_one(username=get_default_user()['username'], req=None)
            if not user:
                get_resource_service('users').post([get_default_user()])
            prepopulate_data(doc.get('profile') + '.json', get_default_user())
        return ['OK']


class AppPrepopulateCommand(superdesk.Command):

    option_list = [
        superdesk.Option('--file', '-f', dest='prepopulate_file', default='app_prepopulate_data.json')
    ]

    def run(self, prepopulate_file):
        user = get_resource_service('users').find_one(username=get_default_user()['username'], req=None)
        if not user:
            get_resource_service('users').post([get_default_user()])
        prepopulate_data(prepopulate_file, get_default_user())


superdesk.command('app:prepopulate', AppPrepopulateCommand())

        # If there is an existing set of renditions we keep those
        if old_item:
            media = old_item.get('renditions', {}).get('original', {}).get('media', {})
            if media:
                item['renditions'] = old_item['renditions']
                item['mimetype'] = old_item.get('mimetype')
                item['filemeta'] = old_item.get('filemeta')
                logger.info("Reuters image not updated for GUID:{}".format(item[GUID_FIELD]))
                return

        content, filename, content_type = download_file_from_url(href)
        file_type, ext = content_type.split('/')

        metadata = process_file(content, file_type)
        file_guid = app.media.put(content, filename, content_type, metadata)
        inserted.append(file_guid)

        rendition_spec = app.config.get('RENDITIONS', {}).get('picture', {})
        renditions = generate_renditions(content, file_guid, inserted, file_type,
                                         content_type, rendition_spec, url_for_media)
        item['renditions'] = renditions
        item['mimetype'] = content_type
        item['filemeta'] = metadata
    except Exception:
        for file_id in inserted:
            app.media.delete(file_id)
        raise


superdesk.command('ingest:update', UpdateIngest())

        if queue_item.get('publish_schedule'):
            publish_schedule = queue_item['publish_schedule']
            if type(publish_schedule) is not datetime:
                raise PublishQueueError.bad_schedule_error(Exception("Schedule is not datetime"),
                                                           destination)
            return utcnow() >= publish_schedule
        return True
    except PublishQueueError:
        raise
    except Exception as ex:
        raise PublishQueueError.bad_schedule_error(ex, destination)


def update_content_state(queue_item):
    """
    Updates the state of the content item to published
    In archive and published repos
    :param queue_item:
    :return:
    """
    if queue_item.get('publish_schedule'):
        try:
            item_update = {'state': 'published'}
            superdesk.get_resource_service('archive').patch(queue_item['item_id'], item_update)
            superdesk.get_resource_service('published').\
                update_published_items(queue_item['item_id'], 'state', 'published')
        except Exception as ex:
            raise PublishQueueError.content_update_error(ex)

superdesk.command('publish:transmit', PublishContent())

    """
    Fetches the expired articles from published collection. Expiry Conditions:
        1.  can_be_removed flag is True
        2.  Item Expiry is less than or equal to expired_date_time, State of the Item is not SCHEDULED and
            allow_post_publish_actions flag is True

    :param expired_date_time:
    :param limit:
    :return: expired articles from published collection
    """

    logger.info('Get expired content from published')
    query = {
        '$or': [
            {'can_be_removed': True},
            {'$and': [
                {'expiry': {'$lte': expired_date_time}},
                {ITEM_STATE: {'$ne': CONTENT_STATE.SCHEDULED}},
                {'allow_post_publish_actions': True}
            ]}
        ]
    }

    req = ParsedRequest()
    req.sort = '_created'
    req.max_results = limit

    return superdesk.get_resource_service('published').get_from_mongo(req=req, lookup=query)

superdesk.command('publish:remove_expired', RemoveExpiredPublishContent())

            with open(local_filepath, 'wb') as file_in_local_storage:
                file_in_local_storage.write(zip_file.read(name))
                extracted_files.append(local_filepath)
        # Save or update the theme in the database
        result = themes_service.save_or_update_theme(description_file, extracted_files, force_update=True)
        return json.dumps(
            dict(
                _status='OK',
                _action=result.get('status'),
                theme=description_file
            ), cls=MongoJSONEncoder)


class ThemesCommand(superdesk.Command):

    def run(self):
        theme_service = get_resource_service('themes')
        created, updated = theme_service.update_registered_theme_with_local_files(force=True)
        print('%d themes registered' % (len(created) + len(updated)))
        if created:
            print('added:')
            for theme in created:
                print('\t+ %s %s (%s)' % (theme.get('label', theme['name']), theme['version'], theme['name']))
        if updated:
            print('updated:')
            for theme in updated:
                print('\t* %s %s (%s)' % (theme.get('label', theme['name']), theme['version'], theme['name']))


superdesk.command('register_local_themes', ThemesCommand())

            logger.info('Inserting {} items'.format(len(items)))
            archive_items = []

            for item in items:
                dest_doc = dict(item)
                new_id = generate_guid(type=GUID_TAG)
                dest_doc[app.config['ID_FIELD']] = new_id
                dest_doc['guid'] = new_id
                generate_unique_id_and_name(dest_doc)

                dest_doc[app.config['VERSION']] = 1
                dest_doc[ITEM_STATE] = CONTENT_STATE.FETCHED
                user_id = desk.get('members', [{'user': None}])[0].get('user')
                dest_doc['original_creator'] = user_id
                dest_doc['version_creator'] = user_id

                from apps.tasks import send_to
                send_to(dest_doc, desk_id=desk_id, stage_id=stage_id, user_id=user_id)
                dest_doc[app.config['VERSION']] = 1  # Above step increments the version and needs to reset
                dest_doc[FAMILY_ID] = item['_id']

                remove_unwanted(dest_doc)
                archive_items.append(dest_doc)

            get_resource_service(ARCHIVE).post(archive_items)
            for item in archive_items:
                insert_into_versions(id_=item[app.config['ID_FIELD']])


superdesk.command('app:scaffold_data', AppScaffoldDataCommand())

                updates = {config.LAST_UPDATED: utcnow()}
                if orig_item.get("retry_attempt", 0) < max_retry_attempt:
                    updates["retry_attempt"] = orig_item.get("retry_attempt", 0) + 1
                    updates["state"] = QueueState.RETRYING.value
                    updates["next_retry_attempt_at"] = utcnow() + timedelta(minutes=retry_attempt_delay)
                else:
                    # all retry attempts exhausted marking the item as failed.
                    updates["state"] = QueueState.FAILED.value

                publish_queue_service.system_update(orig_item.get(config.ID_FIELD), updates, orig_item)
            except:
                logger.error("Failed to set the state for failed publish queue item {}.".format(item_id))

        logger.error("Failed to publish the following items: {}".format(failed_items.keys()))


def can_transmit_queue_item(queue_item):
    """
    Check if the queue item can be tranmitted or not
    :param dict queue_item: queue item
    :return boolean: True or False
    """
    if queue_item.get("state") == QueueState.RETRYING:
        if not queue_item.get("next_retry_attempt_at") <= utcnow():
            return False

    return True


superdesk.command("publish:transmit", PublishContent())

        service = superdesk.get_resource_service(mongo_collection_name)
        req = ParsedRequest()
        req.sort = '[("%s", 1)]' % config.ID_FIELD
        cursor = service.get_from_mongo(req, {})
        count = cursor.count()
        no_of_buckets = len(range(0, count, bucket_size))
        water_mark = cursor[0][config.ID_FIELD]
        print('Number of items to index: {}, pages={}'.format(count, no_of_buckets))
        for x in range(0, no_of_buckets):
            print('{} Page : {}'.format(time.strftime('%X %x %Z'), x + 1))
            s = time.time()
            req = ParsedRequest()
            req.sort = '[("%s", 1)]' % config.ID_FIELD
            req.max_results = bucket_size
            if x == 0:
                lookup = {config.ID_FIELD: {'$gte': water_mark}}
            else:
                lookup = {config.ID_FIELD: {'$gt': water_mark}}

            cursor = service.get_from_mongo(req, lookup)
            items = list(cursor)
            water_mark = items[len(items) - 1][config.ID_FIELD]
            print('{} Retrieved from Mongo in {:.3f} seconds to {}'.format(time.strftime('%X %x %Z'), time.time() - s,
                  water_mark))

            yield items


superdesk.command('app:index_from_mongo', IndexFromMongo())

        items = self._get_children(service, parent)
        items.append(parent)

        for item in items:
            if not self._has_expired(item, expiry_datetime):
                return []
        return items

    def _has_expired(self, item, expiry_datetime):
        """Checks if the item has expired

        :param dict item: The item to check
        :param datetime expiry_datetime: The date and time items should be expired
        :return bool: True if the item has expired, otherwise False
        """
        item.setdefault('expiry', item['_updated'] + timedelta(days=self.expiry_days))
        return item.get('expiry') <= expiry_datetime

    def _get_children(self, service, item):
        """Get the list of children to the root item using the ancestors dictionary key

        :param service: The content_api items service
        :param dict item: The root item to get the children of
        :return list: The list of children for this root item
        """
        return list(service.find({'ancestors': item['_id']}))


superdesk.command('content_api:remove_expired', RemoveExpiredItems())

def enqueue_items(published_items):
    """
    Creates the corresponding entries in the publish queue for each item
    :param list published_items: the list of items marked for publishing
    """
    failed_items = {}
    current_utc = utcnow()

    for queue_item in published_items:
        try:
            schedule_utc_datetime = get_utc_schedule(queue_item, PUBLISH_SCHEDULE)
            if not schedule_utc_datetime or schedule_utc_datetime < current_utc:
                enqueue_item(queue_item)
        except:
            logger.exception('Failed to queue item {}'.format(queue_item.get('_id')))
            failed_items[str(queue_item.get('_id'))] = queue_item

    # mark failed items as pending so that Celery tasks will try again
    if len(failed_items) > 0:
        logger.error('Failed to publish the following items: {}'.format(failed_items.keys()))


superdesk.command('publish:enqueue', EnqueueContent())


@celery.task
def enqueue_published():
    EnqueueContent().run()

    schema = blogs_schema
    datasource = {"source": "blogs", "default_sort": [("title", 1)]}
    resource_methods = ["GET"]


class UserBlogsService(BaseService):
    def get(self, req, lookup):
        if lookup.get("user_id"):
            lookup["members.user"] = ObjectId(lookup["user_id"])
            del lookup["user_id"]
        return super().get(req, lookup)


class PublishBlogsCommand(superdesk.Command):
    """
    Republish blogs on s3 with the right theme
    """

    def run(self):
        # retrieves all opened blogs
        blogs_service = get_resource_service("blogs")
        blogs = blogs_service.get(req=None, lookup=dict(blog_status="open"))
        # republish on s3
        print("\n* Republishing blogs:\n")
        for blog in blogs:
            url = publish_blog_embed_on_s3(blog_id=str(blog["_id"]), safe=False)
            print('  - Blog "%s" republished: %s' % (blog["title"], url))


superdesk.command("publish_blogs", PublishBlogsCommand())

    def get_expired_items(self, now):
        query_filter = self.get_query_for_expired_items(now)
        req = ParsedRequest()
        req.max_results = 100
        req.args = {'filter': query_filter}
        return superdesk.get_resource_service('archive').get(req, None)

    def get_query_for_expired_items(self, now):
        query = {'and':
                 [
                     {'range': {'expiry': {'lte': now}}}
                 ]
                 }
        return superdesk.json.dumps(query)


superdesk.command('archive:remove_expired', ArchiveRemoveExpiredContent())
superdesk.workflow_state('in_progress')
superdesk.workflow_action(
    name='save',
    include_states=['draft', 'fetched', 'routed', 'submitted'],
    privileges=['archive']
)

superdesk.workflow_state('submitted')
superdesk.workflow_action(
    name='move',
    exclude_states=['ingested', 'spiked', 'on-hold', 'published', 'killed'],
    privileges=['archive']
)

    option_list = {
        superdesk.Option('--provider', '-p', dest='provider'),
    }

    def run(self, provider=None):
        if provider:
            data = superdesk.json.loads(provider)
            data.setdefault('_created', utcnow())
            data.setdefault('_updated', utcnow())
            data.setdefault('name', data['type'])
            db = superdesk.get_db()
            db['ingest_providers'].save(data)
            return data

superdesk.command('ingest:update', UpdateIngest())
superdesk.command('ingest:provider', AddProvider())

# load providers now to have available types for the schema
import superdesk.io.reuters
import superdesk.io.aap


def init_app(app):
    IngestProviderModel(app=app)


class IngestProviderModel(BaseModel):
    schema = {
        'name': {
            'type': 'string',

    """Upsert the vocabularies into the vocabularies collections.

    The format of the file used is JSON.
    :param filepath: absolute filepath
    :return: nothing
    """
    if not os.path.exists(filepath):
        raise FileNotFoundError

    [table_name, ext] = os.path.basename(filepath).split('.')

    with open(filepath, 'rt') as vocabularies:
        json_data = json.loads(vocabularies.read())
        populate_table_json(table_name, json_data)


class VocabulariesPopulateCommand(superdesk.Command):
    """
    Class defining the populate vocabularies command.
    """

    option_list = (
        superdesk.Option('--filepath', '-f', dest='filepath', required=True),
    )

    def run(self, filepath):
        process_vocabularies(filepath)


superdesk.command('vocabularies:populate', VocabulariesPopulateCommand())

        :param ad_password: Password of Active Directory Username
        :param username: Username as in Active Directory whose profile needs to be imported to Superdesk.
        :return: User Profile.
        """

        # force type conversion to boolean
        user_type = 'administrator' if admin is not None and admin.lower() == 'true' else 'user'

        # Authenticate and fetch profile from AD
        settings = app.settings
        ad_auth = ADAuth(settings['LDAP_SERVER'], settings['LDAP_SERVER_PORT'], settings['LDAP_BASE_FILTER'],
                         settings['LDAP_USER_FILTER'], settings['LDAP_USER_ATTRIBUTES'], settings['LDAP_FQDN'])

        user_data = ad_auth.authenticate_and_fetch_profile(ad_username, ad_password, username)

        if len(user_data) == 0:
            raise SuperdeskApiError.notFoundError('Username not found')

        # Check if User Profile already exists in Mongo
        user = superdesk.get_resource_service('users').find_one(username=username, req=None)

        if user:
            superdesk.get_resource_service('users').patch(user.get('_id'), user_data)
        else:
            add_default_values(user_data, username, user_type=user_type)
            superdesk.get_resource_service('users').post([user_data])

        return user_data

superdesk.command('users:copyfromad', ImportUserProfileFromADCommand())

        finally:
            mark_task_as_not_running("archive", "update_overdue_scheduled")


def get_overdue_scheduled_items(expired_date_time, resource, limit=100):
    """
    Fetches the overdue scheduled articles from given collection. Overdue Conditions:
        1.  it should be in 'scheduled' state
        2.  publish_schedule is less than or equal to expired_date_time

    :param expired_date_time: DateTime that scheduled tate will be checked against
    :param resource: Name of the resource to check the data from
    :param limit: Number of return items
    :return: overdue scheduled articles from published collection
    """

    logger.info('Get overdue scheduled content from {}'.format(resource))
    query = {'$and': [
        {'publish_schedule': {'$lte': expired_date_time}},
        {ITEM_STATE: CONTENT_STATE.SCHEDULED}
    ]}

    req = ParsedRequest()
    req.sort = '_modified'
    req.max_results = limit

    return superdesk.get_resource_service(resource).get_from_mongo(req=req, lookup=query)

superdesk.command('archive:remove_spiked_if_expired', RemoveExpiredSpikeContent())
superdesk.command('archive:remove_overdue_scheduled', UpdateOverdueScheduledContent())

        if app.config.get('LEGAL_ARCHIVE'):
            legal_archive_items = superdesk.get_resource_service('legal_archive').get_from_mongo(None, query)
            self.__add_existing_files(used_images, legal_archive_items)

            legal_archive_version_items = superdesk.get_resource_service('legal_archive_versions').\
                get_from_mongo(None, query)
            self.__add_existing_files(used_images, legal_archive_version_items)

        print('Number of used files: ', len(used_images))

        superdesk.app.media.remove_unreferenced_files(used_images)

    def __add_existing_files(self, used_images, items):
        for item in items:
            if 'media' in item:
                used_images.add(str(item['media']))

            if item.get('renditions', {}):
                used_images.update([str(rend.get('media')) for rend in item.get('renditions', {}).values()
                                    if rend.get('media')])

            associations = [assoc.get('renditions') for assoc in (item.get(ASSOCIATIONS) or {}).values()
                            if assoc and assoc.get('renditions')]

            for renditions in associations:
                used_images.update([str(rend.get('media')) for rend in renditions.values() if rend.get('media')])


superdesk.command('app:clean_images', CleanImages())

class GetAuthTokenCommand(superdesk.Command):
    """
    Generate an authorization token to be able to authenticate against the REST api without
    starting the client the copy the authorization header.
    """

    option_list = (
        superdesk.Option('--username', '-u', dest='username', required=True),
        superdesk.Option('--password', '-p', dest='password', required=True)
    )

    def run(self, username, password):
        credentials = {
            'username': username,
            'password': password
        }
        service = superdesk.get_resource_service('auth')
        id = str(service.post([credentials])[0])
        print('Session ID:', id)
        creds = service.find_one(req=None, _id=id)
        token = creds.get('token').encode('ascii')
        encoded_token = b'basic ' + b64encode(token + b':')
        print('Generated token: ', encoded_token)
        return encoded_token


superdesk.command('users:create', CreateUserCommand())
superdesk.command('users:hash_passwords', HashUserPasswordsCommand())
superdesk.command('users:get_auth_token', GetAuthTokenCommand())