def generate_remote_synthesis_makefile(self):
        if self.connection.ssh_user == None or self.connection.ssh_server == None:
            p.warning("Connection data is not given. "
                "Accessing environmental variables in the makefile")
        p.info("Generating makefile for remote synthesis.")

        top_mod = self.modules_pool.get_top_module()
        if not os.path.exists(top_mod.fetchto):
            p.warning("There are no modules fetched. "
                "Are you sure it's correct?")

        ise_path = self.__figure_out_ise_path()
        tcl = self.__search_tcl_file()

        if tcl == None:
            self.__generate_tcl()
            tcl = "run.tcl"
        files = self.modules_pool.build_very_global_file_list()

        sff = SourceFileFactory()
        files.add(sff.new(tcl))
        files.add(sff.new(top_mod.syn_project))

        self.make_writer.generate_remote_synthesis_makefile(files=files, name=top_mod.syn_name,
        cwd=os.getcwd(), user=self.connection.ssh_user, server=self.connection.ssh_server, ise_path=ise_path)

def fillDAG_data (jobsub, tag, date, xsec_a_path, outEvents, outRep, outRepSng):
  # check if job is done already
  if isDoneData (tag, date, outRep, outRepSng):
    msg.warning ("xsec validation plots found in " + outRep + " ... " + msg.BOLD + "skipping xsecval:fillDAG_data\n", 1)
    return
  # not done, add jobs to dag
  msg.info ("\tAdding xsec validation (data) jobs\n")    
  # in parallel mode
  jobsub.add ("<parallel>")
  # one job for all comparisons without errors
  inFile  = "file_list-" + tag + "-" + date + ".xml"
  outFile = "genie_" + tag + "-" + date + "-world_nu_xsec_data_comp-all-withref"
  cmd = "gvld_nu_xsec -g input/" + inFile + " -o " + outFile
  # add the command to dag
  inputs = outRep + "/" + inFile + " " + xsec_a_path + "/xsec-vA-" + tag + ".root " + outEvents + "/*.ghep.root"
  logFile = "gvld_nu_xsec_all.log"
  jobsub.addJob (inputs, outRep, logFile, cmd)
  # job per comparison with error
  for comp in comparisons:
    outFile = "genie_" + tag + "-" + date + "-world_nu_xsec_data_comp-" + comp
    cmd = "gvld_nu_xsec -e -g input/" + inFile + " -o " + outFile + " -c " + comp
    logFile = "gvld_nu_xsec_" + comp + ".log"
    jobsub.addJob (inputs, outRepSng, logFile, cmd)
  # done
  jobsub.add ("</parallel>")

def match(input1, input2, output, key):

    """Check if matching dataset is the same for inputs. Copy to output.

    input1 -- input file
    input2 -- input file
    output -- output file
    key -- dataset to match
    """

    if key not in input1 or key not in input2:
        msg.error("Both files must contains %s" % key)
        sys.exit(1)

    if len(input1[key].shape) != 1 or len(input2[key].shape) != 1:
        msg.error("Matching key should have (N,) shape.")
        sys.exit(1)

    if not np.array_equal(input1[key], input2[key]):
        msg.error("%s in input files are not the same." % key)
        sys.exit(1)

    msg.info("Copying %s" % key)

    input1.copy(key, output)

def fillDAG_sanity (jobsub, events, out):
  # check if job is done already
  if isDoneSanity (out):
    msg.warning ("Standard mctest sanity checks log files found in " + out + " ... " + msg.BOLD + \
                 "skipping standard:fillDAG_sanity\n", 1)
    return
  # not done, add jobs to dag
  msg.info ("\tAdding mctest sanity checks jobs\n")
  # in parallel mode
  jobsub.add ("<parallel>")
  # common options
  options = " --add-event-printout-in-error-log --event-record-print-level 2 --max-num-of-errors-shown 10 " + \
            " --check-energy-momentum-conservation " + \
            " --check-charge-conservation " + \
            " --check-for-pseudoparticles-in-final-state " + \
            " --check-for-off-mass-shell-particles-in-final-state " + \
            " --check-for-num-of-final-state-nucleons-inconsistent-with-target " + \
            " --check-vertex-distribution " + \
            " --check-decayer-consistency"
  # loop over keys and generate gvld_sample_scan command
  for key in nuPDG.iterkeys():
    inputFile = "gntp." + key + ".ghep.root"
    output = "gntp." + key + ".ghep.root.sanity.log"
    cmd = "gvld_sample_scan -f input/" + inputFile + " -o " + output + options
    logFile = "gvld_sample_scan." + key + ".log"
    jobsub.addJob (events + "/" + inputFile, out, logFile, cmd)
  # done
  jobsub.add ("</parallel>")

 def submit(self):
   self.dag.close()
   msg.info ("Done with dag file. Ready to submit.\n")
   # check if run is not empty
   if os.stat(self.dagFile).st_size == 0:
     msg.warning ("Dag file: " + self.dagFile + " is empty. " + msg.RED + msg.BOLD + "NO JOBS TO RUN!!!\n")
     exit (0)
   # submit dag
   msg.info ("Submitting: " + self.dagFile + "\n")
   subprocess.Popen (self.setup + self.subdag, shell=True, executable="/bin/bash")

 def clean_modules(self):
     p.info("Removing fetched modules..")
     remove_list = [m for m in self.modules_pool if m.source in ["svn", "git"] and m.isfetched]
     remove_list.reverse() #we will remove modules in backward order
     if len(remove_list):
         for m in remove_list:
             p.rawprint("\t" + m.url + " [from: " + m.path + "]")
             m.remove_dir_from_disk()
     else:
         p.info("There are no modules to be removed")

def copy(source, output, keys):

    """Copy selected datasets.

    Keyword arguments:
    source -- input file
    output -- output file
    keys -- datasets to be copied
    """

    for k in keys:
        msg.info("Copying %s" % k)
        source.copy(k, output)

    def generate_quartus_project(self):
        p.info("Generating/updating Quartus project.")

        if not self.modules_pool.is_everything_fetched():
            p.error("A module remains unfetched. "
                "Fetching must be done prior to makefile generation")
            p.rawprint(str([str(m) for m in self.modules_pool.modules if not m.isfetched]))
            quit()

        if os.path.exists(self.top_module.syn_project + ".qsf"):
            self.__update_existing_quartus_project()
        else:
            self.__create_new_quartus_project()

    def generate_isim_makefile(self):
#        p.info("Generating makefile for simulation.")
        p.info("Generating ISE Simulation (ISim) makefile for simulation.")
        solver = DependencySolver()

        pool = self.modules_pool
        if not pool.is_everything_fetched():
            p.echo("A module remains unfetched. "
                "Fetching must be done prior to makefile generation. Try issuing \"hdlmake2 --fetch\"")
            p.echo(str([str(m) for m in self.modules_pool.modules if not m.isfetched]))
            quit()
        top_module = pool.get_top_module()
        flist = pool.build_global_file_list();
        flist_sorted = solver.solve(flist);
        self.make_writer.generate_isim_makefile(flist_sorted, top_module)

 def generate_ise_project(self):
     p.info("Generating/updating ISE project")
     if self.__is_xilinx_screwed():
         p.error("Xilinx environment variable is unset or is wrong.\n"
             "Cannot generate ise project")
         quit()
     if not self.modules_pool.is_everything_fetched():
         p.echo("A module remains unfetched. Fetching must be done prior to makefile generation")
         p.echo(str([str(m) for m in self.modules_pool if not m.isfetched]))
         quit()
     ise = self.__check_ise_version()
     if os.path.exists(self.top_module.syn_project):
         self.__update_existing_ise_project(ise=ise)
     else:
         self.__create_new_ise_project(ise=ise)

def copy(source, output, keys):

    """Copy requested datasets.

    Keyword arguments:
    source -- input file
    output -- output file
    keys -- keys to be copied
    """

    for k in keys:
        if k not in source:
            msg.warning("%s requested, but not found." % k)
            continue
        else:
            msg.info("Copying %s" % k)
            source.copy(k, output)

def fillDAG_GST (jobsub, out):
  # check if job is done already
  if isDoneGST (out):
    msg.warning ("xsec validation gst files found in " + out + " ... " + msg.BOLD + "skipping xsecval:fillDAG_GST\n", 1)
    return
  # not done, add jobs to dag
  msg.info ("\tAdding xsec validation (gst) jobs\n")
  # in parallel mode
  jobsub.add ("<parallel>")
  # loop over keys and generate gntpc command
  for key in nuPDG.iterkeys():
    inputFile = "gntp." + key + ".ghep.root"
    logFile = "gntpc" + key + ".log"
    cmd = "gntpc -f gst -i input/" + inputFile
    jobsub.addJob (out + "/" + inputFile, out, logFile, cmd)
  # done
  jobsub.add ("</parallel>")

def fillDAG_data (jobsub, tag, date, xsec_n_path, outEvents, outRep):
  # check if job is done already
  if isDoneData (tag, date, outRep):
    msg.warning ("hadronization test plots found in " + outRep + " ... " + msg.BOLD + "skipping hadronization:fillDAG_data\n", 1)
    return
  # not done, add jobs to dag
  msg.info ("\tAdding hadronization test (plots) jobs\n")    
  # in serial mode
  jobsub.add ("<serial>")
  inFile  = "file_list-" + tag + "-" + date + ".xml"
  outFile = "genie_" + tag + "-hadronization_test.ps"
  cmd = "gvld_hadronz_test -g input/" + inFile + " -o " + outFile
  # add the command to dag
  inputs = outRep + "/" + inFile + " " + xsec_n_path + "/xsec-vN-" + tag + ".root " + outEvents + "/*.ghep.root"
  logFile = "gvld_hadronz_test.log"
  jobsub.addJob (inputs, outRep, logFile, cmd)
  # done
  jobsub.add ("</serial>")

    def run(self):
        p.info("Running automatic flow")

        tm = self.top_module

        if not self.modules_pool.is_everything_fetched():
            self.fetch(unfetched_only = True)

        if tm.action == "simulation":
            # Defaults to isim simulator tool
            if global_mod.sim_tool == "isim":
                self.generate_isim_makefile()
            elif global_mod.sim_tool == "vsim":
                self.generate_vsim_makefile()
            else:
                raise RuntimeError("Unrecognized or not specified simulation tool: "+ str(global_mod.sim_tool))
                quit()
            # Force declaration of sim_tool varible in Manifest
            #if tm.sim_tool == None:
            #	p.error("sim_tool variable must be defined in the manifest")
            #	quit()
            ## Make distintion between isim and vsim simulators
            #if tm.sim_tool == "vsim":
            #       	self.generate_vsim_makefile()
            #elif tm.sim_tool == "isim":
            #	self.generate_isim_makefile()
            #else:
            #	raise RuntimeError("Unrecognized sim tool: "+tm.sim_tool)
        elif tm.action == "synthesis":
            if tm.syn_project == None:
                p.error("syn_project variable must be defined in the manfiest")
                quit()
            if tm.target.lower() == "xilinx":
                self.generate_ise_project()
                self.generate_ise_makefile()
                self.generate_remote_synthesis_makefile()
            elif tm.target.lower() == "altera":
                self.generate_quartus_project()
#                self.generate_quartus_makefile()
#                self.generate_quartus_remote_synthesis_makefile()
            else:
                raise RuntimeError("Unrecognized target: "+tm.target)
        else:
            p.print_action_help() and quit()

def fillDAGEv (jobsub, tag, xsec_a_path, out):
  # check if job is done already
  if isDoneEv (out):
    msg.warning ("Repeatability test events found in " + out + " ... " + msg.BOLD + "skipping reptest:fillDAGEv\n", 1)
    return
  # not done, add jobs to dag
  msg.info ("\tAdding repeatability test (gevgen) jobs\n")
  # in parallel mode
  jobsub.add ("<parallel>")
  # common options
  inputFile = "gxspl-vA-" + tag + ".xml"
  options = " -p 14 -t 1000260560 -e 0.1,50 -f 1/x --seed 123456 --cross-sections input/" + inputFile
  # loop over runs and generate gevgen command
  for run in runs:
    cmd = "gevgen " + options + " -r " + run
    logFile = "gevgen_" + run + ".log"
    jobsub.addJob (xsec_a_path + "/" + inputFile, out, logFile, cmd)
  # done
  jobsub.add ("</parallel>")

def fillDAGPart (jobsub, tag, out):
  # check if job is done already
  if isDonePart (out):
    msg.warning ("Nucleons splines found in " + out + " ... " + msg.BOLD + "skipping nun:fillDAGPart\n", 1)
    return
  # not done, add jobs to dag
  msg.info ("\tAdding nucleon splines (part) jobs\n")
  # in parallel mode
  jobsub.add ("<parallel>")
  # common options
  inputs = "none"
  # loop over keys and generate proper command
  for key in nuPDG.iterkeys():
    cmd = "gmkspl -p " + nuPDG[key] + " -t " + targetPDG[key] + " -n " + nKnots + " -e " + maxEnergy \
          + " -o " + outXML[key] + " --event-generator-list " + generatorList[key]
    logFile = "gmkspl." + outXML[key] + ".log"
    jobsub.addJob (inputs, out, logFile, cmd)
  # done
  jobsub.add ("</parallel>")

def fillDAG_GHEP (jobsub, tag, xsec_a_path, out):
  # check if job is done already
  if isDoneGHEP (out):
    msg.warning ("xsec validation ghep files found in " + out + " ... " + msg.BOLD + "skipping xsecval:fillDAG_GHEP\n", 1)
    return
  #not done, add jobs to dag
  msg.info ("\tAdding xsec validation (ghep) jobs\n")
  # in parallel mode
  jobsub.add ("<parallel>")
  # common configuration
  inputFile = "gxspl-vA-" + tag + ".xml"
  options   = " -n " + nEvents + " -e " + energy + " -f " + flux + " --seed " + mcseed + \
              " --cross-sections input/" + inputFile + " --event-generator-list " + generatorList
  # loop over keys and generate gevgen command
  for key in nuPDG.iterkeys():
    cmd = "gevgen " + options + " -p " + nuPDG[key] + " -t " + targetPDG[key] + " -r " + key
    logFile = "gevgen_" + key + ".log"
    jobsub.addJob (xsec_a_path + "/" + inputFile, out, logFile, cmd)
  # done
  jobsub.add ("</parallel>")

def fillDAGTest (jobsub, events, out):
  # check if job is done already
  if isDoneTest (out):
    msg.warning ("Repeatability test logs found in " + out + " ... " + msg.BOLD + "skipping reptest:fillDAGTest\n", 1)
    return
  # not done, add jobs to dag
  msg.info ("\tAdding repeatability test (gvld) jobs\n")
  # in parallel mode
  jobsub.add ("<parallel>")
  # common options
  options = " --add-event-printout-in-error-log --max-num-of-errors-shown 10 "
  input1 = "gntp." + runs[0] + ".ghep.root" 
  # loop over runs and generate proper command
  for run in runs[1:]:
    input2 = "gntp." + run + ".ghep.root"
    output = "reptest_runs" + runs[0] + "vs" + run + ".log"
    logFile = "gvld_repeatability_test_" + runs[0] + "vs" + run + ".log"
    cmd = "gvld_repeatability_test --first-sample input/" + input1 + \
          " --second-sample input/" + input2 + options + " -o " + output
    jobsub.addJob (events + "/*.ghep.root", out, logFile, cmd)
  # done  
  jobsub.add ("</parallel>")

def fillDAGPart (jobsub, tag, xsec_n_path, out):
  # check if job is done already
  if isDonePart (tag, out):
    msg.warning ("Nucleus splines found in " + out + " ... " + msg.BOLD + "skipping nua:fillDAGPart\n", 1)
    return
  # not done, add jobs to dag
  msg.info ("\tAdding nucleus splines (part) jobs\n")
  # in parallel mode
  jobsub.add ("<parallel>")
  # common options
  inputFile = "gxspl-vN-" + tag + ".xml"
  inputs = xsec_n_path + "/*.xml"
  options = " --input-cross-sections input/" + inputFile
  # loop over targets and generate proper command
  for t in targets:
    outputFile = "gxspl_" + t + ".xml"
    cmd = "gmkspl -p " + nuPDG + " -t " + t + " -n " + nKnots + " -e " + maxEnergy + options + \
          " --output-cross-sections " + outputFile
    logFile = "gxspl_" + t + ".xml.log"
    jobsub.addJob (inputs, out, logFile, cmd)
  # done
  jobsub.add ("</parallel>")

def get_data(filename, match, keys):

    """Load file, check if contains match,
    update datasets based on command line options. Return data dictionary.

    Keyword arguments:
    filename -- input hdf5 file
    match -- common key use to order data
    keys -- user-chosen datasets to save
    """

    data = hdf5.load(filename)

    print "\nThe following datasets were found in %s:\n" % filename
    msg.list_dataset(data)

    check.key_exists(match, data, filename)

    if keys:
        msg.info("Using only: " + keys)
        update_data(data, [k.strip() for k in keys.split(',')], args.match)

    return data