def test_choice(self):
		print("Verifying directory linking mechanism")
		print(nt.dirNameProxy["Kurogane"]["fqPath"], nt.getCanonicalMangaUpdatesName(nt.dirNameProxy["Kurogane"]["fqPath"]))
		print(nt.dirNameProxy["Kyoumen no Silhouette"]["fqPath"], nt.getCanonicalMangaUpdatesName(nt.dirNameProxy["Kyoumen no Silhouette"]["fqPath"]))
		print(nt.dirNameProxy["Neko Ane "]["fqPath"], nt.getCanonicalMangaUpdatesName(nt.dirNameProxy["Neko Ane "]["fqPath"]))
		print(nt.dirNameProxy["Rescue Me"]["fqPath"], nt.getCanonicalMangaUpdatesName(nt.dirNameProxy["Rescue Me"]["fqPath"]))
		print(nt.dirNameProxy["Maken Ki!"]["fqPath"], nt.getCanonicalMangaUpdatesName(nt.dirNameProxy["Maken Ki!"]["fqPath"]))
		print(nt.dirNameProxy[":REverSAL"]["fqPath"], nt.getCanonicalMangaUpdatesName(nt.dirNameProxy[":REverSAL"]["fqPath"]))
		print(nt.dirNameProxy["Silva"]["fqPath"], nt.getCanonicalMangaUpdatesName(nt.dirNameProxy["Silva"]["fqPath"]))
		print(nt.dirNameProxy["Kouya ni Kemono Doukokusu"]["fqPath"], nt.getCanonicalMangaUpdatesName(nt.dirNameProxy["Kouya ni Kemono Doukokusu"]["fqPath"]))
		print(nt.dirNameProxy["Koukaku no Regios - Missing Mail"]["fqPath"], nt.getCanonicalMangaUpdatesName(nt.dirNameProxy["Koukaku no Regios - Missing Mail"]["fqPath"]))
		print(nt.dirNameProxy["Kuraudo (NOUJOU Junichi) "]["fqPath"], nt.getCanonicalMangaUpdatesName(nt.dirNameProxy["Kuraudo (NOUJOU Junichi) "]["fqPath"]))

def consolicateSeriesToSingleDir():
	print("Looking for series directories that can be flattened to a single dir")
	idLut = nt.MtNamesMapWrapper("buId->fsName")
	db = DbInterface()
	for key, luDict in nt.dirNameProxy.iteritems():
		# print("Key = ", key)
		mId = db.getIdFromDirName(key)

		# Skip cases where we have no match
		if not mId:
			continue

		dups = set()
		for name in idLut[mId]:
			cName = nt.prepFilenameForMatching(name)

			# Skip if it's one of the manga names that falls apart under the directory name cleaning mechanism
			if not cName:
				continue

			if cName in nt.dirNameProxy:
				dups.add(cName)
				db.getIdFromDirName(cName)
		if len(dups) > 1:
			row = db.getRowByValue(buId=mId)
			targetName = nt.prepFilenameForMatching(row["buName"])
			dest = nt.dirNameProxy[targetName]
			if luDict["dirKey"] != targetName and dest["fqPath"]:
				print("baseName = ", row["buName"], ", id = ", mId, ", names = ", dups)

				print("	URL: https://www.mangaupdates.com/series.html?id=%s" % (mId, ))
				print(" Dir 1 ", luDict["fqPath"])
				print(" Dir 2 ", dest["fqPath"])

				dirName = os.path.split(luDict["fqPath"])[-1]
				dir2Name = os.path.split(dest["fqPath"])[-1]

				print("	1:	", dirName, ' ->', nt.getCanonicalMangaUpdatesName(dirName))
				print("	2:	", dir2Name, ' ->', nt.getCanonicalMangaUpdatesName(dir2Name))
				print("	1:	({num} items)".format(num=len(os.listdir(luDict["fqPath"]))))
				print("	2:	({num} items)".format(num=len(os.listdir(dest["fqPath"]))))


				doMove = query_response("move files ('f' dir 1 -> dir 2. 'r' dir 2 -> dir 1. 'n' do not move)?")
				if doMove == "forward":
					moveFiles(luDict["fqPath"], dest["fqPath"])
					os.rmdir(luDict["fqPath"])
				elif doMove == "reverse":
					moveFiles(dest["fqPath"], luDict["fqPath"])
					os.rmdir(dest["fqPath"])

	def getChaptersFromSeriesPage(self, inUrl):

		soup = self.wg.getSoup(inUrl)

		if 'The following content is intended for mature' in soup.get_text():
			self.log.info("Adult check page. Confirming...")
			soup = self.wg.getSoup(inUrl, postData={"adult": "true"})

		mainDiv = soup.find('div', id='series_right')

		seriesName = mainDiv.h1.get_text()

		seriesName = nt.getCanonicalMangaUpdatesName(seriesName)

		# No idea why chapters are class 'staff_link'. Huh.
		chapters = mainDiv.find_all('div', class_='staff_link')


		ret = []
		for chapter in chapters:
			item = {}
			item['originName'] = "{series} - {file}".format(series=seriesName, file=chapter.a.get_text())
			item['sourceUrl']  = chapter.a['href']
			item['seriesName'] = seriesName
			item['retreivalTime'] = time.time()    # Fukkit, just use the current date.
			ret.append(item)
		return ret

	def loadRemoteDirectory(self, fullPath, aggregate=False):
		ret = {}

		for dirName, stats in self.ftp.mlsd(fullPath):

			# Skip items that aren't directories
			if stats["type"]!="dir":
				continue

			canonName = nt.getCanonicalMangaUpdatesName(dirName)
			matchingName = nt.prepFilenameForMatching(canonName)

			fqPath = os.path.join(fullPath, dirName)

			# matchName = os.path.split(ret[matchingName])[-1]

			if matchingName in ret:
				# if aggregate:
				# 	fqPath = self.aggregateDirs(fullPath, dirName, matchName)
				# else:
				if COMPLAIN_ABOUT_DUPS:
					self.log.warning("Duplicate directories for series '%s'!", canonName)
					self.log.warning("	'%s'", dirName)
					self.log.warning("	'%s'", matchingName)
				ret[matchingName] = fqPath

			else:
				ret[matchingName] = fqPath

		return ret

	def getDoujinshiUploadDirectory(self, seriesName):
		ulDir = self.getExistingDir(seriesName)

		if not ulDir:
			seriesName = nt.getCanonicalMangaUpdatesName(seriesName)
			safeFilename = nt.makeFilenameSafe(seriesName)
			matchName = nt.prepFilenameForMatching(seriesName)
			matchName = matchName.encode('latin-1', 'ignore').decode('latin-1')

			self.checkInitDirs()
			if matchName in self.unsortedDirs:
				ulDir = self.unsortedDirs[matchName]
			elif safeFilename in self.unsortedDirs:
				ulDir = self.unsortedDirs[safeFilename]
			else:

				self.log.info("Need to create container directory for %s", seriesName)
				ulDir = os.path.join(settings.mkSettings["uploadContainerDir"], settings.mkSettings["uploadDir"], safeFilename)
				try:
					self.sftp.mkdir(ulDir)
				except ftplib.error_perm:
					self.log.warn("Directory exists?")
					self.log.warn(traceback.format_exc())


		return ulDir

	def updateSeriesDbEntryById(self, rowId, commit=True, **kwargs):

		# Patch series name.
		if "seriesName" in kwargs and kwargs["seriesName"]:
			kwargs["seriesName"] = nt.getCanonicalMangaUpdatesName(kwargs["seriesName"])

		queries = []
		qArgs = []
		for key in kwargs.keys():
			if key not in self.validSeriesKwargs:
				raise ValueError("Invalid keyword argument: %s" % key)
			else:
				queries.append("{k}=%s".format(k=key))
				qArgs.append(kwargs[key])

		qArgs.append(rowId)

		column = ", ".join(queries)


		query = '''UPDATE {tableName} SET {v} WHERE dbId=%s;'''.format(tableName=self.seriesTableName, v=column)

		if QUERY_DEBUG:
			print("Query = ", query)
			print("Args = ", qArgs)

		with self.conn.cursor() as cur:

			if commit:
				cur.execute("BEGIN;")

			cur.execute(query, qArgs)

			if commit:
				cur.execute("COMMIT;")

	def getUploadDirectory(self, seriesName):

		ulDir = self.getExistingDir(seriesName)

		if not ulDir:
			seriesName   = nt.getCanonicalMangaUpdatesName(seriesName)
			safeFilename = nt.makeFilenameSafe(seriesName)
			matchName    = nt.prepFilenameForMatching(seriesName)
			matchName    = matchName.encode('utf-8', 'ignore').decode('utf-8')

			self.checkInitDirs()
			if matchName in self.mainDirs:
				ulDir = self.mainDirs[matchName][0]
			elif seriesName in self.mainDirs:
				ulDir = self.mainDirs[seriesName][0]
			else:

				self.log.info("Need to create container directory for %s", seriesName)
				ulDir = os.path.join(settings.mkSettings["uploadContainerDir"], settings.mkSettings["uploadDir"], safeFilename)
				try:
					self.sftp.mkdir(ulDir)
				except OSError as e:
					# If the error is just a "directory exists" warning, ignore it silently
					if str(e) == 'OSError: File already exists':
						pass
					else:
						self.log.warn("Error creating directory?")
						self.log.warn(traceback.format_exc())


		return ulDir

	def getUploadDirectory(self, seriesName):

		ulDir = self.getExistingDir(seriesName)

		if not ulDir:
			seriesName = nt.getCanonicalMangaUpdatesName(seriesName)
			safeFilename = nt.makeFilenameSafe(seriesName)
			matchName = nt.prepFilenameForMatching(seriesName)
			matchName = matchName.encode('latin-1', 'ignore').decode('latin-1')

			self.checkInitDirs()
			if matchName in self.unsortedDirs:
				ulDir = self.unsortedDirs[matchName]
			elif safeFilename in self.unsortedDirs:
				ulDir = self.unsortedDirs[safeFilename]
			else:

				self.log.info("Need to create container directory for %s", seriesName)
				ulDir = os.path.join(settings.mkSettings["uploadContainerDir"], settings.mkSettings["uploadDir"], safeFilename)
				try:
					self.ftp.mkd(ulDir)
				except ftplib.error_perm as e:
					# If the error is just a "directory exists" warning, ignore it silently
					if str(e).startswith("550") and str(e).endswith('File exists'):
						pass
					else:
						self.log.warn("Error creating directory?")
						self.log.warn(traceback.format_exc())


		return ulDir

	def getItemsFromContainer(self, dirName, dirUrl):

		# Skip the needs sorting directory.
		if dirName == 'Needs sorting':
			return [], []
		if dirName == 'Admin Cleanup':
			return [], []
		if dirName == 'Raws':
			return [], []
		if dirName == 'Requests':
			return [], []
		if dirName == '_Autouploads':
			return [], []


		self.log.info("Original name - %s", dirName)

		bracketStripRe = re.compile(r"(\[.*?\])")
		dirName = bracketStripRe.sub(" ", dirName)
		while dirName.find("  ")+1:
			dirName = dirName.replace("  ", " ")
		dirName = dirName.strip()

		if not dirName:
			self.log.critical("Empty dirname = '%s', baseURL = '%s'", dirName, dirUrl)
			raise ValueError("No dir name for directory!")

		dirName = nt.getCanonicalMangaUpdatesName(dirName)

		self.log.info("Canonical name - %s", dirName)
		self.log.info("Fetching items for directory '%s'", dirName)

		self.log.info("Using URL '%s'", dirUrl)
		try:
			itemPage = self.wg.getpage(dirUrl)
		except urllib.error.URLError:
			self.log.error("Could not fetch page '%s'", dirUrl)
			return [], []

		soup = bs4.BeautifulSoup(itemPage)




		itemRet = []
		dirRet  = []

		for row in soup.find_all("tr"):

			dirDat, itemDat = self.parseRow(row, dirUrl, dirName)

			if dirDat:
				dirRet.append(dirDat)

			if itemDat:
				itemRet.append(itemDat)


		return dirRet, itemRet

	def extractFilename(self, inString):
		title, dummy_blurb = inString.rsplit("|", 1)
		# title, chapter = title.rsplit("-", 1)

		# Unescape htmlescaped items in the name/chapter
		ps = html.parser.HTMLParser()
		title = ps.unescape(title)

		vol = None
		chap = None
		volChap = None

		try:
			if " vol " in title.lower():
				title, volChap = title.rsplit(" vol ", 1)
				vol, dummy = volChap.strip().split(" ", 1)
		except ValueError:
			self.log.error("Could not parse volume number from title %s", title)
			traceback.print_exc()


		try:
			if volChap and " ch " in volChap:
				dummy, chap = volChap.rsplit(" ch ", 1)

			elif " ch " in title:
				title, chap = title.rsplit(" ch ", 1)

		except ValueError:
			self.log.error("Could not parse chapter number from title %s", title)
			traceback.print_exc()

		if chap:
			if "Page" in chap:
				chap, dummy = chap.split("Page", 1)

		elif title and "Page" in title:
			title, dummy = title.split("Page", 1)

		title = title.rstrip(" -")
		# haveLookup = nt.haveCanonicalMangaUpdatesName(title)
		# if not haveLookup:
		# 	self.log.warning("Did not find title '%s' in MangaUpdates database!", title)
		title = nt.getCanonicalMangaUpdatesName(title).strip()


		volChap = []

		if vol:
			volChap.append("v{}".format(vol))
		if chap:
			volChap.append("c{}".format(chap))

		chapter = " ".join(volChap)

		return title, chapter.strip()

	def aggregateDirs(self, pathBase_1, pathBase_2, dir1, dir2):
		canonName    = nt.getCanonicalMangaUpdatesName(dir1)
		canonNameAlt = nt.getCanonicalMangaUpdatesName(dir2)
		cname1 = nt.prepFilenameForMatching(canonName)
		cname2 = nt.prepFilenameForMatching(canonNameAlt)
		if canonName.lower() != canonNameAlt.lower():
			self.log.critical("Error in uploading file. Name lookup via MangaUpdates table not commutative!")
			self.log.critical("First returned value    '%s'", canonName)
			self.log.critical("For directory with path '%s'", dir1)
			self.log.critical("Second returned value   '%s'", canonNameAlt)
			self.log.critical("For directory with path '%s'", dir2)
			self.log.critical("After cleaning: '%s', '%s', equal: '%s'", cname1, cname2, cname1 == cname2)


			raise CanonMismatch("Identical and yet not? '%s' - '%s'" % (canonName, canonNameAlt))
		self.log.info("Aggregating directories for canon name '%s':", canonName)

		n1 = lv.distance(dir1, canonName)
		n2 = lv.distance(dir2, canonName)

		self.log.info("	%s - '%s'", n1, dir1)
		self.log.info("	%s - '%s'", n2, dir2)

		# I'm using less then or equal, so situations where
		# both names are equadistant get aggregated anyways.
		if n1 <= n2:
			src = os.path.join(pathBase_2, dir2)
			dst = os.path.join(pathBase_1, dir1)
		else:
			src = os.path.join(pathBase_1, dir1)
			dst = os.path.join(pathBase_2, dir2)


		self.moveItemsInDir(src, dst)
		self.log.info("Removing directory '%s'", src)
		try:
			self.sftp.mkdir("/Admin cleanup/autoclean dirs")
		except:
			pass
		self.sftp.rename(src, "/Admin cleanup/autoclean dirs/garbage dir %s" % src.replace("/", ";").replace(" ", "_"))

		return dst

    def aggregateDirs(self, pathBase, dir1, dir2):
        canonName = nt.getCanonicalMangaUpdatesName(dir1)
        canonNameAlt = nt.getCanonicalMangaUpdatesName(dir2)
        if canonName.lower() != canonNameAlt.lower():
            self.log.critical(
                "Error in uploading file. Name lookup via MangaUpdates table not commutative!"
            )
            self.log.critical("First returned value    '%s'", canonName)
            self.log.critical("For directory with path '%s'", dir1)
            self.log.critical("Second returned value   '%s'", canonNameAlt)
            self.log.critical("For directory with path '%s'", dir2)

            raise ValueError("Identical and yet not? '%s' - '%s'" %
                             (canonName, canonNameAlt))
        self.log.info("Aggregating directories for canon name '%s':",
                      canonName)

        n1 = lv.distance(dir1, canonName)
        n2 = lv.distance(dir2, canonName)

        self.log.info("	%s - '%s'", n1, dir1)
        self.log.info("	%s - '%s'", n2, dir2)

        # I'm using less then or equal, so situations where
        # both names are equadistant get aggregated anyways.
        if n1 <= n2:
            src = dir2
            dst = dir1
        else:
            src = dir1
            dst = dir2

        src = os.path.join(pathBase, src)
        dst = os.path.join(pathBase, dst)

        self.moveItemsInDir(src, dst)
        self.log.info("Removing directory '%s'", src)
        # self.ftp.rmd(src)
        # self.ftp.rename(src, "/Admin Cleanup/garbage dir %s" % id(src))

        return dst

	def getSeries(self, markup):
		soup = bs4.BeautifulSoup(markup, "lxml")
		title = soup.find("h3", id='chapter-title')

		if title.b.find('a'):
			title = title.b.a.get_text()

		else:
			title = title.b.get_text()

		title = nt.getCanonicalMangaUpdatesName(title)
		print("Title '%s'" % title)
		return title

    def updateDbEntryById(self, rowId, commit=True, **kwargs):

        # Patch series name.
        if "seriesName" in kwargs and kwargs["seriesName"] and self.shouldCanonize:
            kwargs["seriesName"] = nt.getCanonicalMangaUpdatesName(kwargs["seriesName"])

        query, queryArguments = self.generateUpdateQuery(dbId=rowId, **kwargs)

        if self.QUERY_DEBUG:
            print("Query = ", query)
            print("Args = ", queryArguments)

        with self.conn.cursor() as cur:
            with transaction(cur, commit=commit):
                cur.execute(query, queryArguments)

	def getFeed(self):
		treedata = self.wg.getJson(self.tree_api)
		assert 'contents' in treedata
		assert treedata['name'] == 'mango'
		assert treedata['type'] == 'directory'
		data_unfiltered = self.process_tree_elements(treedata['contents'])

		data = []
		for sName, filen in data_unfiltered:
			assert filen.startswith(STRIP_PREFIX)
			filen = filen[len(STRIP_PREFIX):]
			if not any([filen.startswith(prefix) for prefix in MASK_PATHS]):
				sName = nt.getCanonicalMangaUpdatesName(sName)
				data.append((sName, filen))

		return data

	def processLinksIntoDB(self, linksDicts):

		self.log.info( "Inserting...",)
		newItems = 0
		for link in linksDicts:
			if link is None:
				print("linksDicts", linksDicts)
				print("WAT")

			row = self.getRowsByValue(sourceUrl=link["sourceUrl"], limitByKey=False)

			if not row:
				newItems += 1

				if not "dlState" in link:
					link['dlState'] = 0

				# Patch series name.
				if 'seriesName' in link and self.shouldCanonize:
					link["seriesName"] = nt.getCanonicalMangaUpdatesName(link["seriesName"])


				# Using fancy dict hijinks now. Old call below for reference.

				# self.insertIntoDb(retreivalTime = link["date"],
				# 					sourceUrl   = link["dlLink"],
				# 					originName  = link["dlName"],
				# 					dlState     = 0,
				# 					seriesName  = link["baseName"],
				# 					flags       = flagStr)

				self.insertIntoDb(**link)


				self.log.info("New item: %s", link)




		self.log.info( "Done")
		self.log.info( "Committing...",)
		self.conn.commit()
		self.log.info( "Committed")

		return newItems

	def findIfMigrated(self, filePath):
		dirPath, fileName = os.path.split(filePath)

		series = dirPath.split("/")[-1]
		series = nt.getCanonicalMangaUpdatesName(series)
		otherDir = nt.dirNameProxy[series]

		if not otherDir["fqPath"]:
			return False
		if otherDir["fqPath"] == dirPath:
			return False

		newPath = os.path.join(otherDir["fqPath"], fileName)
		if os.path.exists(newPath):
			print("File moved!")
			return otherDir["fqPath"]

		return False

	def getDownloadPath(self, item, fName):

		if not item['seriesName']:
			self.log.info("No series set for item. Guessing from filename:")
			self.log.info("Filename = '%s'", fName)
			bareName = nt.guessSeriesFromFilename(fName)

			# if not matchName or not matchName in nt.dirNameProxy:
			if not nt.haveCanonicalMangaUpdatesName(bareName):
				item["seriesName"] = settings.ircBot["unknown-series"]
			else:
				item["seriesName"] = nt.getCanonicalMangaUpdatesName(bareName)

			self.log.info("Guessed  = '%s'. Updating series information", item['seriesName'])
			self.updateDbEntry(item["sourceUrl"], seriesName=item["seriesName"])


		dlPath, newDir = self.locateOrCreateDirectoryForSeries(item["seriesName"])

		if item["flags"] == None:
			item["flags"] = ""

		if newDir:
			self.updateDbEntry(item["sourceUrl"], flags=" ".join([item["flags"], "haddir"]))
			self.conn.commit()

		fqFName = os.path.join(dlPath, fName)

		loop = 1

		fName, ext = os.path.splitext(fName)

		while os.path.exists(fqFName):
			fName = "%s - (%d).%s" % (fName, loop, ext)
			fqFName = os.path.join(dlPath, fName)
			loop += 1
		self.log.info("Saving to archive = %s", fqFName)


		self.updateDbEntry(item["sourceUrl"], downloadPath=dlPath, fileName=fName, originName=fName)

		return fqFName

    def updateDbEntry(self, sourceUrl, commit=True, **kwargs):

        # Patch series name.
        if "seriesName" in kwargs and kwargs["seriesName"] and self.shouldCanonize:
            kwargs["seriesName"] = nt.getCanonicalMangaUpdatesName(kwargs["seriesName"])

            # Clamp the retreivaltime to now, so parsing issues that result in invalid, future
            # time-stamps don't cause posts to stick to the top of the post list.
        if "retreivalTime" in kwargs:
            if kwargs["retreivalTime"] > time.time():
                kwargs["retreivalTime"] = time.time()

        query, queryArguments = self.generateUpdateQuery(sourceUrl=sourceUrl, **kwargs)

        if self.QUERY_DEBUG:
            print("Query = ", query)
            print("Args = ", queryArguments)

        with self.conn.cursor() as cur:
            with transaction(cur, commit=commit):
                cur.execute(query, queryArguments)

    def locateOrCreateDirectoryForSeries(self, seriesName):

        if self.shouldCanonize:
            canonSeriesName = nt.getCanonicalMangaUpdatesName(seriesName)
        else:
            canonSeriesName = seriesName

        safeBaseName = nt.makeFilenameSafe(canonSeriesName)

        if canonSeriesName in nt.dirNameProxy:
            self.log.info(
                "Have target dir for '%s' Dir = '%s'", canonSeriesName, nt.dirNameProxy[canonSeriesName]["fqPath"]
            )
            return nt.dirNameProxy[canonSeriesName]["fqPath"], False
        else:
            self.log.info("Don't have target dir for: %s, full name = %s", canonSeriesName, seriesName)
            targetDir = os.path.join(settings.baseDir, safeBaseName)
            if not os.path.exists(targetDir):
                try:
                    os.makedirs(targetDir)
                    return targetDir, True

                except FileExistsError:
                    # Probably means the directory was concurrently created by another thread in the background?
                    self.log.critical("Directory doesn't exist, and yet it does?")
                    self.log.critical(traceback.format_exc())
                    pass
                except OSError:
                    self.log.critical("Directory creation failed?")
                    self.log.critical(traceback.format_exc())

            else:
                self.log.warning("Directory not found in dir-dict, but it exists!")
                self.log.warning("Directory-Path: %s", targetDir)
                self.log.warning("Base series name: %s", seriesName)
                self.log.warning("Canonized series name: %s", canonSeriesName)
                self.log.warning("Safe canonized name: %s", safeBaseName)
            return targetDir, False