def readpaths(tracks):
    paths = []
    logger.debug("Reading {0} total tracks".format(len(tracks)))

    for label, track, attributes in tracks:
        path = Path()
        path.label = session.query(Label).get(label)
        
        logger.debug("Received a {0} track".format(path.label.text))

        for frame, userbox in track.items():
            box = Box(path = path)
            box.xtl = int(userbox[0])
            box.ytl = int(userbox[1])
            box.xbr = int(userbox[2])
            box.ybr = int(userbox[3])
            box.occluded = int(userbox[4])
            box.outside = int(userbox[5])
            box.frame = int(frame)

            logger.debug("Received box {0}".format(str(box.getbox())))

        for attributeid, timeline in attributes.items():
            attribute = session.query(Attribute).get(attributeid)
            for frame, value in timeline.items():
                aa = AttributeAnnotation()
                aa.attribute = attribute
                aa.frame = frame
                aa.value = value
                path.attributes.append(aa)

        paths.append(path)
    return paths

    def __call__(self, args):
        video = session.query(Video).filter(Video.slug == args.slug)
        if not video.count():
            print "Video {0} does not exist!".format(args.slug)
            return
        video = video.one()

        query = session.query(Path)
        query = query.join(Job)
        query = query.join(Segment)
        query = query.filter(Segment.video == video)
        numpaths = query.count()
        if numpaths and not args.force:
            print ("Video has {0} paths. Use --force to delete."
                .format(numpaths))
            return

        for segment in video.segments:
            for job in segment.jobs:
                if job.published and not job.completed:
                    hitid = job.disable()
                    print "Disabled {0}".format(hitid)

        session.delete(video)
        session.commit()

        print "Deleted video and associated data."

    def __call__(self, args):
        query = session.query(HIT)
        query = query.filter(HIT.useful == True)
        if args.hit:
            query = query.filter(HIT.hitid == args.id)
        else:
            worker = session.query(Worker).get(args.id)
            if not worker:
                print "Worker \"{0}\" not found".format(args.id)
                return
            if not args.no_block:
                worker.block("HIT was invalid.")
                print "Blocked worker \"{0}\"".format(args.id)
                session.add(worker)

            query = query.filter(HIT.workerid == args.id)

        for hit in query:
            replacement = hit.invalidate()
            session.add(hit)
            print "Invalidated {0}".format(hit.hitid)

            if replacement:
                session.add(replacement)
                if not args.no_publish:
                    session.commit()
                    replacement.publish()
                    session.add(replacement)
                    print "Respawned with {0}".format(replacement.hitid)
        session.commit()

    def __call__(self, args):
        videos = session.query(Video)

        if args.training:
            videos = videos.filter(Video.isfortraining == True)
        else:
            videos = videos.filter(Video.isfortraining == False)
            if args.worker:
                videos = videos.join(Segment)
                videos = videos.join(Job)
                videos = videos.filter(Job.workerid == args.worker)
            elif args.published:
                videos = videos.join(Segment)
                videos = videos.join(Job)
                videos = videos.filter(Job.published == True)
            elif args.completed:
                videos = videos.join(Segment)
                videos = videos.join(Job)
                videos = videos.filter(Job.completed == True)
            elif args.incomplete:
                videos = videos.join(Segment)
                videos = videos.join(Job)
                videos = videos.filter(Job.completed == False)
        
        if args.count:
            print videos.count()
        else:
	    newvideos = dict()
	    print "Identifier", '-------', 'jobid', '------', 'timeonserver', ' ------------------- HitId', ' ------------------- AssignmentId', ' --------------- WorkerId'
            for video in videos.distinct():
#                print video.slug
		# Print videos sorted by time
		test = session.query(Job).filter(Job.useful == True)
		test = test.join(Segment).filter(Segment.videoid == video.id)
		if test.count() != 1:
		    print "Error: ", test.count()
		    break
		for t in test:
		    l = list()
		    if t.timeonserver is None:
			l.append(datetime.datetime.now())
		    else:
			l.append(t.timeonserver)
		    l.append(t)
		    newvideos[video.slug] = l
	    sorted_list = [x for x in newvideos.iteritems()]
	    sorted_list.sort(key=lambda x: x[1][0]) # sort by key
	    for k in sorted_list:
		print k[0], "---", k[1][1].id, "---", k[1][0], "---", k[1][1].hitid, "---", k[1][1].assignmentid, "---", k[1][1].workerid

def validatejob(id, data):
    job = session.query(Job).get(id)
    paths = readpaths(data["tracks"])
    predicates = readpredicates(data["predicates"])

    return (job.trainingjob.validator(paths, job.trainingjob.paths) and
            job.trainingjob.validator(predicates, job.trainingjob.predicates))

def getallvideos():
    query = session.query(Video)
    videos = []
    for video in query:
        newvideo = {
            "slug": video.slug,
            "segments": [],
        }
        for segment in video.segments:
            newsegment = {
                "start": segment.start,
                "stop":segment.stop,
                "jobs":[],
            }

            for job in segment.jobs:
                newsegment["jobs"].append({
                    "url": job.offlineurl(config.localhost),
                    "numobjects": len(job.paths),
                    "numdone": len([path for path in job.paths if path.done]),
                })

            newvideo["segments"].append(newsegment)

        videos.append(newvideo)
    return videos

def trackforward(id, frame, tracker, trackid, tracks):
    frame = int(frame)
    trackid = int(trackid)
    job = session.query(Job).get(id)
    segment = job.segment
    video = segment.video
    paths = [path for path in readpaths(tracks) if path is not None]
    paths = trackutils.totrackpaths(paths)

    logger.info("Job Id: {0}".format(id))
    logger.info("Algorithm: {0}".format(tracker))

    start = frame
    stop = segment.stop
    
    outpath = tracking.api.online(tracker, start, stop, video.location, trackid, paths)
    path = trackutils.fromtrackpath(outpath, job, start, stop)
    attrs = [(x.attributeid, x.frame, x.value) for x in path.attributes]

    logger.info("Path: {0}".format(path))

    return {
        "label": 0,
        "boxes": [tuple(x) for x in path.getboxes()],
        "attributes": attrs
    }

def savetopview(slug, image, environ):
    logger.info("Saving topview image")

    query = session.query(Video).filter(Video.slug == slug)
    if query.count() != 1:
        raise ValueError("Invalid video slug")
    video = query[0]

    savedir = video.homographylocation
    if savedir is None:
        savedir = makehomographydir(video)

    savelocation = os.path.join(savedir, "topview.jpg")
    tempformfile = tempfile.TemporaryFile()
    tempformfile.write(image)
    tempformfile.seek(0)
    form = cgi.FieldStorage(fp=tempformfile, environ=environ, keep_blank_values=True)
    outfile = open(savelocation, "w+b")
    shutil.copyfileobj(form['photo'].file, outfile)
    tempformfile.close()
    outfile.close()

    newimage = cv2.imread(savelocation)
    scale = 1
    if newimage.shape[1] > video.width:
        scale = float(video.width) / float(newimage.shape[1])
        newimage = cv2.resize(newimage, (0, 0), None, scale, scale)

    if newimage.shape[0] > video.height:
        scale = float(video.height) / float(newimage.shape[0])
        newimage = cv2.resize(newimage, (0, 0), None, scale, scale)

    cv2.imwrite(savelocation, newimage)

    def __call__(self, args):
        videos = session.query(Video)

        if args.training:
            videos = videos.filter(Video.isfortraining == True)
        else:
            videos = videos.filter(Video.isfortraining == False)
            if args.worker:
                videos = videos.join(Segment)
                videos = videos.join(Job)
                videos = videos.filter(Job.workerid == args.worker)
            elif args.published:
                videos = videos.join(Segment)
                videos = videos.join(Job)
                videos = videos.filter(Job.published == True)
            elif args.completed:
                videos = videos.join(Segment)
                videos = videos.join(Job)
                videos = videos.filter(Job.completed == True)
        
        if args.count:
            print videos.count()
        else:
            for video in videos.distinct():
                print video.slug

def videodump(slug, outputtype, groundplane, fields=None):
    logger.debug(os.getcwd())
    query = session.query(Video).filter(Video.slug == slug)
    if query.count() != 1:
        raise ValueError("Invalid video slug")
    video = query.one()

    #mergemethod = merge.userid
    groundplane = (groundplane == 1)
    mergemethod = merge.getpercentoverlap(groundplane)
    if fields is None:
        if groundplane:
            fields = dumptools.GROUND_PLANE_FORMAT
        else:
            fields = dumptools.DEFAULT_FORMAT
    fields = fields.split()

    data = dumptools.getdata(video, True, mergemethod, 0.5, None, groundplane)

    outfile = tempfile.TemporaryFile()
    if outputtype == "json":
        dumptools.dumpjson(outfile, data, groundplane, fields)
    elif outputtype == "xml":
        dumptools.dumpxml(outfile, data, groundplane, fields)
    else:
        dumptools.dumptext(outfile, data, groundplane, fields)

    outfile.seek(0)
    text = outfile.readlines()
    outfile.close()
    return text

def upload(id, environ):
    job = session.query(Job).get(id)
    data = environ["wsgi.input"]

    # read meta data first
    header = data.readline().strip() + "--"
    while True:
        chunk = data.readline()
        if chunk.strip() == "":
            break
        key, value = chunk.split(": ", 1)
        if key == "Content-Type":
            job.mimetype = value.strip()
        if key == "Content-Disposition":
            for item in value.split("; "):
                itemdata = item.split("=", 1)
                if len(itemdata) == 2 and itemdata[0] == "filename":
                    job.filename = itemdata[1].strip()[1:-1]
    session.commit()

    # now read the file data, looking for the terminating sequence
    out = open(job.storepath, "wb")
    while True:
        chunk = data.readline(1024 * 1024)
        if chunk.strip() == header:
            break
        out.write(chunk)
    out.close()

    return ["<script>parent.uploaded();</script>"]

def savejob(id, data):
    job = session.query(Job).get(id)

    for path in job.paths:
        session.delete(path)
    for pi in job.predicate_instances:
        for pa in pi.predicate_annotations:
            session.delete(pa)
        session.delete(pi)
    for s in job.sentences:
        for sa in s.annotations:
            session.delete(sa)
        session.delete(s)
    session.commit()
    
    paths = readpaths(data["tracks"])
    for path in paths:
        job.paths.append(path)
    for pi in readpredicates(data["predicates"], paths):
        job.predicate_instances.append(pi)
    for s in readsentences(data['sentences']):
        job.sentences.append(s)
    
    session.add(job)
    session.commit()

 def __call__(self, args):
     query = session.query(Job).filter(turkic.models.HIT.completed == True)
     for job in query:
         print job.storepath
         print job.mimetype
         for activity in job.activities:
             print activity.text
         print ""

def markcomplete(hitid, assignmentid, workerid):
    """
    Marks a job as complete. Usually this is called right before the
    MTurk form is submitted.
    """
    hit = session.query(models.HIT).filter(models.HIT.hitid == hitid).one()
    hit.markcompleted(workerid, assignmentid)
    session.add(hit)
    session.commit()

def getboxesforjob(id):
    job = session.query(Job).get(id)
    result = []
    for path in job.paths:
        attrs = [(x.attributeid, x.frame, x.value) for x in path.attributes]
        result.append({"label": path.labelid,
                       "boxes": [tuple(x) for x in path.getboxes()],
                       "attributes": attrs})
    return result

def savedonationstatus(hitid, donation):
    """
    Saves the donation statistics
    """
    hit = session.query(models.HIT).filter(models.HIT.hitid == hitid).one()
    hit.opt2donate = float(donation)
    hit.opt2donate = min(max(hit.opt2donate, 0), 1)

    session.add(hit)
    session.commit()

def getsentenceannotationsforjob(id):
    job = session.query(Job).get(id)
    results = []
    for s in job.sentences:
        annotations = [(a.frame, a.value) for a in s.annotations]
        results.append({
            'sentence': s.text,
            'annotations': annotations,
        })
    return results

def getjob(id, verified):
    job = session.query(Job).get(id)

    logger.debug("Found job {0}".format(job.id))

    if int(verified) and job.segment.video.trainwith:
        # swap segment with the training segment
        training = True
        segment = job.segment.video.trainwith.segments[0]
        logger.debug("Swapping actual segment with training segment")
    else:
        training = False
        segment = job.segment

    video = segment.video
    labels = dict((l.id, l.text) for l in video.labels)

    attributes = {}
    for label in video.labels:
        attributes[label.id] = dict((a.id, a.text) for a in label.attributes)

    roles = dict((r.id, r.text) for r in session.query(Role))
    predicates = dict((p.id, p.text) for p in session.query(Predicate))

    logger.debug("Giving user frames {1} to {2} of {0}".format(video.slug,
                                                               segment.start,
                                                               segment.stop))

    return {"start":        segment.start,
            "stop":         segment.stop,
            "slug":         video.slug,
            "width":        video.width,
            "height":       video.height,
            "skip":         video.skip,
            "perobject":    video.perobjectbonus,
            "completion":   video.completionbonus,
            "blowradius":   video.blowradius,
            "jobid":        job.id,
            "training":     int(training),
            "labels":       labels,
            "attributes":   attributes,
            "roles":        roles,
            "predicates":   predicates}

def readpath(label, userid, done, track, attributes):
    path = Path()
    path.label = session.query(Label).get(label)
    path.done = int(done)
    path.userid = int(userid)
 
    logger.debug("Received a {0} track".format(path.label.text))

    visible = False
    for frame, userbox in track.items():
        box = Box(path = path)
        #box.xtl = max(int(userbox[0]), 0)
        #box.ytl = max(int(userbox[1]), 0)
        #box.xbr = max(int(userbox[2]), 0)
        #box.ybr = max(int(userbox[3]), 0)
        box.xtl = int(userbox[0])
        box.ytl = int(userbox[1])
        box.xbr = int(userbox[2])
        box.ybr = int(userbox[3])

        box.occluded = int(userbox[4])
        box.outside = int(userbox[5])
        box.generated = int(userbox[6])
        box.frame = int(frame)
        if not box.outside:
            visible = True

        logger.debug("Received box {0}".format(str(box.getbox())))

    if not visible:
        logger.warning("Received empty path! Skipping")
        return

    for attributeid, timeline in attributes.items():
        attribute = session.query(Attribute).get(attributeid)
        for frame, value in timeline.items():
            aa = AttributeAnnotation()
            aa.attribute = attribute
            aa.frame = frame
            aa.value = value
            path.attributes.append(aa)
    return path

def savejob(id, tracks):
    job = session.query(Job).get(id)

    for path in job.paths:
        session.delete(path)
    session.commit()
    for path in readpaths(tracks):
        job.paths.append(path)

    session.add(job)
    session.commit()