def deleteFiles(self):
        self.log.debug("Deleting files from content.json...")
        files = []  # Get filenames
        for content_inner_path in self.site.content_manager.contents.keys():
            content = self.site.content_manager.contents[content_inner_path]
            files.append(content_inner_path)
            # Add normal files
            for file_relative_path in content.get("files", {}).keys():
                file_inner_path = helper.getDirname(content_inner_path) + file_relative_path  # Relative to site dir
                files.append(file_inner_path)
            # Add optional files
            for file_relative_path in content.get("files_optional", {}).keys():
                file_inner_path = helper.getDirname(content_inner_path) + file_relative_path  # Relative to site dir
                files.append(file_inner_path)

        if self.isFile("dbschema.json"):
            self.log.debug("Deleting db file...")
            self.closeDb()
            self.has_db = False
            try:
                schema = self.loadJson("dbschema.json")
                db_path = self.getPath(schema["db_file"])
                if os.path.isfile(db_path):
                    os.unlink(db_path)
            except Exception, err:
                self.log.error("Db file delete error: %s" % err)

    def verifyFiles(self, quick_check=False):  # Fast = using file size
        bad_files = []

        if not self.site.content_manager.contents.get("content.json"):  # No content.json, download it first
            self.site.needFile("content.json", update=True)  # Force update to fix corrupt file
            self.site.content_manager.loadContent()  # Reload content.json
        for content_inner_path, content in self.site.content_manager.contents.items():
            if not os.path.isfile(self.getPath(content_inner_path)):  # Missing content.json file
                self.log.debug("[MISSING] %s" % content_inner_path)
                bad_files.append(content_inner_path)

            for file_relative_path in content.get("files", {}).keys():
                file_inner_path = helper.getDirname(content_inner_path) + file_relative_path  # Relative to site dir
                file_inner_path = file_inner_path.strip("/")  # Strip leading /
                file_path = self.getPath(file_inner_path)
                if not os.path.isfile(file_path):
                    self.log.debug("[MISSING] %s" % file_inner_path)
                    bad_files.append(file_inner_path)
                    continue

                if quick_check:
                    ok = os.path.getsize(file_path) == content["files"][file_relative_path]["size"]
                else:
                    ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb"))

                if not ok:
                    self.log.debug("[CHANGED] %s" % file_inner_path)
                    bad_files.append(file_inner_path)

            # Optional files
            optional_added = 0
            optional_removed = 0
            for file_relative_path in content.get("files_optional", {}).keys():
                file_inner_path = helper.getDirname(content_inner_path) + file_relative_path  # Relative to site dir
                file_inner_path = file_inner_path.strip("/")  # Strip leading /
                file_path = self.getPath(file_inner_path)
                if not os.path.isfile(file_path):
                    self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"])
                    continue

                if quick_check:
                    ok = os.path.getsize(file_path) == content["files_optional"][file_relative_path]["size"]
                else:
                    ok = self.site.content_manager.verifyFile(file_inner_path, open(file_path, "rb"))

                if ok:
                    self.site.content_manager.hashfield.appendHash(content["files_optional"][file_relative_path]["sha512"])
                    optional_added += 1
                else:
                    self.site.content_manager.hashfield.removeHash(content["files_optional"][file_relative_path]["sha512"])
                    optional_removed += 1
                    self.log.debug("[OPTIONAL CHANGED] %s" % file_inner_path)

            self.log.debug(
                "%s verified: %s, quick: %s, bad: %s, optionals: +%s -%s" %
                (content_inner_path, len(content["files"]), quick_check, bad_files, optional_added, optional_removed)
            )

        return bad_files

    def loadContent(self, content_inner_path="content.json", add_bad_files=True, delete_removed_files=True, load_includes=True):
        content_inner_path = content_inner_path.strip("/")  # Remove / from begning
        old_content = self.contents.get(content_inner_path)
        content_path = self.site.storage.getPath(content_inner_path)
        content_dir = helper.getDirname(self.site.storage.getPath(content_inner_path))
        content_inner_dir = helper.getDirname(content_inner_path)

        if os.path.isfile(content_path):
            try:
                new_content = json.load(open(content_path))
            except Exception, err:
                self.log.error("%s load error: %s" % (content_path, Debug.formatException(err)))
                return [], []

 def getDiffs(self, inner_path, limit=30 * 1024, update_files=True):
     if inner_path not in self.contents:
         return None
     diffs = {}
     content_inner_path_dir = helper.getDirname(inner_path)
     for file_relative_path in self.contents[inner_path].get("files", {}):
         file_inner_path = content_inner_path_dir + file_relative_path
         if self.site.storage.isFile(file_inner_path + "-new"):  # New version present
             diffs[file_relative_path] = Diff.diff(
                 list(self.site.storage.open(file_inner_path)),
                 list(self.site.storage.open(file_inner_path + "-new")),
                 limit=limit
             )
             if update_files:
                 self.site.storage.delete(file_inner_path)
                 self.site.storage.rename(file_inner_path + "-new", file_inner_path)
         if self.site.storage.isFile(file_inner_path + "-old"):  # Old version present
             diffs[file_relative_path] = Diff.diff(
                 list(self.site.storage.open(file_inner_path + "-old")),
                 list(self.site.storage.open(file_inner_path)),
                 limit=limit
             )
             if update_files:
                 self.site.storage.delete(file_inner_path + "-old")
     return diffs

    def actionBigfileUploadInit(self, to, inner_path, size):
        valid_signers = self.site.content_manager.getValidSigners(inner_path)
        auth_address = self.user.getAuthAddress(self.site.address)
        if not self.site.settings["own"] and auth_address not in valid_signers:
            self.log.error("FileWrite forbidden %s not in valid_signers %s" % (auth_address, valid_signers))
            return self.response(to, {"error": "Forbidden, you can only modify your own files"})

        nonce = CryptHash.random()
        piece_size = 1024 * 1024
        inner_path = self.site.content_manager.sanitizePath(inner_path)
        file_info = self.site.content_manager.getFileInfo(inner_path, new_file=True)

        content_inner_path_dir = helper.getDirname(file_info["content_inner_path"])
        file_relative_path = inner_path[len(content_inner_path_dir):]

        upload_nonces[nonce] = {
            "added": time.time(),
            "site": self.site,
            "inner_path": inner_path,
            "websocket_client": self,
            "size": size,
            "piece_size": piece_size,
            "piecemap": inner_path + ".piecemap.msgpack"
        }
        return {
            "url": "/ZeroNet-Internal/BigfileUpload?upload_nonce=" + nonce,
            "piece_size": piece_size,
            "inner_path": inner_path,
            "file_relative_path": file_relative_path
        }

 def getPiecemap(self, inner_path):
     file_info = self.site.content_manager.getFileInfo(inner_path)
     piecemap_inner_path = helper.getDirname(file_info["content_inner_path"]) + file_info["piecemap"]
     self.site.needFile(piecemap_inner_path, priority=20)
     piecemap = msgpack.unpack(self.site.storage.open(piecemap_inner_path))[helper.getFilename(inner_path)]
     piecemap["piece_size"] = file_info["piece_size"]
     return piecemap

    def getUserContentRules(self, parent_content, inner_path, content):
        user_contents = parent_content["user_contents"]

        # Delivered for directory
        if "inner_path" in parent_content:
            parent_content_dir = helper.getDirname(parent_content["inner_path"])
            user_address = re.match("([A-Za-z0-9]*?)/", inner_path[len(parent_content_dir):]).group(1)
        else:
            user_address = re.match(".*/([A-Za-z0-9]*?)/.*?$", inner_path).group(1)

        try:
            if not content:
                content = self.site.storage.loadJson(inner_path)  # Read the file if no content specified
            user_urn = "%s/%s" % (content["cert_auth_type"], content["cert_user_id"])  # web/[email protected]
            cert_user_id = content["cert_user_id"]
        except Exception:  # Content.json not exist
            user_urn = "n-a/n-a"
            cert_user_id = "n-a"

        if user_address in user_contents["permissions"]:
            rules = copy.copy(user_contents["permissions"].get(user_address, {}))  # Default rules based on address
        else:
            rules = copy.copy(user_contents["permissions"].get(cert_user_id, {}))  # Default rules based on username

        if rules is False:
            banned = True
            rules = {}
        else:
            banned = False
        if "signers" in rules:
            rules["signers"] = rules["signers"][:]  # Make copy of the signers
        for permission_pattern, permission_rules in user_contents["permission_rules"].items():  # Regexp rules
            if not SafeRe.match(permission_pattern, user_urn):
                continue  # Rule is not valid for user
            # Update rules if its better than current recorded ones
            for key, val in permission_rules.iteritems():
                if key not in rules:
                    if type(val) is list:
                        rules[key] = val[:]  # Make copy
                    else:
                        rules[key] = val
                elif type(val) is int:  # Int, update if larger
                    if val > rules[key]:
                        rules[key] = val
                elif hasattr(val, "startswith"):  # String, update if longer
                    if len(val) > len(rules[key]):
                        rules[key] = val
                elif type(val) is list:  # List, append
                    rules[key] += val

        rules["cert_signers"] = user_contents["cert_signers"]  # Add valid cert signers
        if "signers" not in rules:
            rules["signers"] = []

        if not banned:
            rules["signers"].append(user_address)  # Add user as valid signer
        rules["user_address"] = user_address
        rules["includes_allowed"] = False

        return rules

    def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}):
        s = time.time()
        if config.verbose:
            self.log.debug("Downloading %s..." % inner_path)
        found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
        content_inner_dir = helper.getDirname(inner_path)
        if not found:
            self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications))
            if check_modifications:  # Download failed, but check modifications if its succed later
                self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications")
            return False  # Could not download content.json

        if config.verbose:
            self.log.debug("Got %s" % inner_path)
        changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False)

        if peer:  # Update last received update from peer to prevent re-sending the same update to it
            peer.last_content_json_update = self.content_manager.contents[inner_path]["modified"]

        # Start download files
        file_threads = []
        if download_files:
            for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
                file_inner_path = content_inner_dir + file_relative_path

                # Try to diff first
                diff_success = False
                diff_actions = diffs.get(file_relative_path)
                if diff_actions and self.bad_files.get(file_inner_path):
                    try:
                        new_file = Diff.patch(self.storage.open(file_inner_path, "rb"), diff_actions)
                        new_file.seek(0)
                        diff_success = self.content_manager.verifyFile(file_inner_path, new_file)
                        if diff_success:
                            self.log.debug("Patched successfully: %s" % file_inner_path)
                            new_file.seek(0)
                            self.storage.write(file_inner_path, new_file)
                            self.onFileDone(file_inner_path)
                    except Exception, err:
                        self.log.debug("Failed to patch %s: %s" % (file_inner_path, err))
                        diff_success = False

                if not diff_success:
                    # Start download and dont wait for finish, return the event
                    res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
                    if res is not True and res is not False:  # Need downloading and file is allowed
                        file_threads.append(res)  # Append evt

            # Optionals files
            if inner_path == "content.json":
                gevent.spawn(self.updateHashfield)

            if self.settings.get("autodownloadoptional"):
                for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys():
                    file_inner_path = content_inner_dir + file_relative_path
                    # Start download and dont wait for finish, return the event
                    res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
                    if res is not True and res is not False:  # Need downloading and file is allowed
                        file_threads.append(res)  # Append evt

    def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False):
        s = time.time()
        self.log.debug("Downloading %s..." % inner_path)
        found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
        content_inner_dir = helper.getDirname(inner_path)
        if not found:
            self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications))
            if check_modifications:  # Download failed, but check modifications if its succed later
                self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications")
            return False  # Could not download content.json

        self.log.debug("Got %s" % inner_path)
        changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False)

        # Start download files
        file_threads = []
        if download_files:
            for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
                file_inner_path = content_inner_dir + file_relative_path
                # Start download and dont wait for finish, return the event
                res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
                if res is not True and res is not False:  # Need downloading and file is allowed
                    file_threads.append(res)  # Append evt

            # Optionals files
            if inner_path == "content.json":
                gevent.spawn(self.updateHashfield)

            if self.settings.get("autodownloadoptional"):
                for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys():
                    file_inner_path = content_inner_dir + file_relative_path
                    # Start download and dont wait for finish, return the event
                    res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
                    if res is not True and res is not False:  # Need downloading and file is allowed
                        file_threads.append(res)  # Append evt

        # Wait for includes download
        include_threads = []
        for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
            file_inner_path = content_inner_dir + file_relative_path
            include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
            include_threads.append(include_thread)

        self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
        gevent.joinall(include_threads)
        self.log.debug("%s: Includes download ended" % inner_path)

        if check_modifications:  # Check if every file is up-to-date
            self.checkModifications(0)

        self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
        gevent.joinall(file_threads)
        self.log.debug("%s: DownloadContent ended in %.2fs" % (inner_path, time.time() - s))

        if not self.worker_manager.tasks:
            self.onComplete()  # No more task trigger site complete

        return True

 def listContents(self, inner_path="content.json", user_files=False):
     if inner_path not in self.contents:
         return []
     back = [inner_path]
     content_inner_dir = helper.getDirname(inner_path)
     for relative_path in self.contents[inner_path].get("includes", {}).keys():
         include_inner_path = content_inner_dir + relative_path
         back += self.listContents(include_inner_path)
     return back

    def getFileInfo(self, inner_path, new_file=False):
        dirs = inner_path.split("/")  # Parent dirs of content.json
        inner_path_parts = [dirs.pop()]  # Filename relative to content.json
        while True:
            content_inner_path = "%s/content.json" % "/".join(dirs)
            content_inner_path = content_inner_path.strip("/")
            content = self.contents.get(content_inner_path)

            # Check in files
            if content and "files" in content:
                back = content["files"].get("/".join(inner_path_parts))
                if back:
                    back["content_inner_path"] = content_inner_path
                    back["optional"] = False
                    back["relative_path"] = "/".join(inner_path_parts)
                    return back

            # Check in optional files
            if content and "files_optional" in content:  # Check if file in this content.json
                back = content["files_optional"].get("/".join(inner_path_parts))
                if back:
                    back["content_inner_path"] = content_inner_path
                    back["optional"] = True
                    back["relative_path"] = "/".join(inner_path_parts)
                    return back

            # Return the rules if user dir
            if content and "user_contents" in content:
                back = content["user_contents"]
                content_inner_path_dir = helper.getDirname(content_inner_path)
                relative_content_path = inner_path[len(content_inner_path_dir):]
                user_auth_address_match = re.match("([A-Za-z0-9]+)/.*", relative_content_path)
                if user_auth_address_match:
                    user_auth_address = user_auth_address_match.group(1)
                    back["content_inner_path"] = "%s%s/content.json" % (content_inner_path_dir, user_auth_address)
                else:
                    back["content_inner_path"] = content_inner_path_dir + "content.json"
                back["optional"] = None
                back["relative_path"] = "/".join(inner_path_parts)
                return back

            if new_file and content:
                back = {}
                back["content_inner_path"] = content_inner_path
                back["relative_path"] = "/".join(inner_path_parts)
                back["optional"] = None
                return back

            # No inner path in this dir, lets try the parent dir
            if dirs:
                inner_path_parts.insert(0, dirs.pop())
            else:  # No more parent dirs
                break

        # Not found
        return False

 def deleteContent(self, site, inner_path):
     content = site.content_manager.contents.get(inner_path)
     if content and "files_optional" in content:
         site_id = self.site_ids[site.address]
         content_inner_dir = helper.getDirname(inner_path)
         optional_inner_paths = [
             content_inner_dir + relative_inner_path
             for relative_inner_path in content.get("files_optional", {}).keys()
         ]
         self.execute("DELETE FROM file_optional WHERE ?", {"site_id": site_id, "inner_path": optional_inner_paths})
     super(ContentDbPlugin, self).deleteContent(site, inner_path)

 def removeContent(self, inner_path):
     inner_dir = helper.getDirname(inner_path)
     try:
         content = self.contents[inner_path]
         files = dict(
             content.get("files", {}),
             **content.get("files_optional", {})
         )
     except Exception, err:
         self.log.debug("Error loading %s for removeContent: %s" % (inner_path, Debug.formatException(err)))
         files = {}

 def testGetDirname(self):
     assert helper.getDirname("data/users/content.json") == "data/users/"
     assert helper.getDirname("data/users") == "data/"
     assert helper.getDirname("") == ""
     assert helper.getDirname("content.json") == ""
     assert helper.getDirname("data/users/") == "data/users/"
     assert helper.getDirname("/data/users/content.json") == "/data/users/"

    def actionSiteSign(self, to, privatekey=None, inner_path="content.json", *args, **kwargs):
        # Add file to content.db and set it as pinned
        content_db = self.site.content_manager.contents.db
        content_inner_dir = helper.getDirname(inner_path)
        content_db.my_optional_files[self.site.address + "/" + content_inner_dir] = time.time()
        if len(content_db.my_optional_files) > 50:  # Keep only last 50
            oldest_key = min(
                content_db.my_optional_files.iterkeys(),
                key=(lambda key: content_db.my_optional_files[key])
            )
            del content_db.my_optional_files[oldest_key]

        return super(UiWebsocketPlugin, self).actionSiteSign(to, privatekey, inner_path, *args, **kwargs)

    def loadContent(self, content_inner_path="content.json", add_bad_files=True, delete_removed_files=True, load_includes=True, force=False):
        content_inner_path = content_inner_path.strip("/")  # Remove / from beginning
        old_content = self.contents.get(content_inner_path)
        content_path = self.site.storage.getPath(content_inner_path)
        content_dir = helper.getDirname(self.site.storage.getPath(content_inner_path))
        content_inner_dir = helper.getDirname(content_inner_path)

        if os.path.isfile(content_path):
            try:
                # Check if file is newer than what we have
                if not force and old_content and not self.site.settings.get("own"):
                    for line in open(content_path):
                        if '"modified"' not in line:
                            continue
                        match = re.search("([0-9\.]+),$", line.strip(" \r\n"))
                        if match and float(match.group(1)) <= old_content.get("modified", 0):
                            self.log.debug("%s loadContent same json file, skipping" % content_inner_path)
                            return [], []

                new_content = json.load(open(content_path))
            except Exception, err:
                self.log.error("%s load error: %s" % (content_path, Debug.formatException(err)))
                return [], []

 def setContent(self, site, inner_path, content, size=0):
     super(ContentDbPlugin, self).setContent(site, inner_path, content, size=size)
     old_content = site.content_manager.contents.get(inner_path, {})
     if (not self.need_filling or self.filled.get(site.address)) and "files_optional" in content or "files_optional" in old_content:
         self.setContentFilesOptional(site, inner_path, content)
         # Check deleted files
         if old_content:
             old_files = old_content.get("files_optional", {}).keys()
             new_files = content.get("files_optional", {}).keys()
             content_inner_dir = helper.getDirname(inner_path)
             deleted = [content_inner_dir + key for key in old_files if key not in new_files]
             if deleted:
                 site_id = self.site_ids[site.address]
                 self.execute("DELETE FROM file_optional WHERE ?", {"site_id": site_id, "inner_path": deleted})

 def removeContent(self, inner_path):
     inner_dir = helper.getDirname(inner_path)
     content = self.contents[inner_path]
     files = dict(
         content.get("files", {}),
         **content.get("files_optional", {})
     )
     files["content.json"] = True
     # Deleting files that no longer in content.json
     for file_relative_path in files:
         file_inner_path = inner_dir + file_relative_path
         try:
             self.site.storage.delete(file_inner_path)
             self.log.debug("Deleted file: %s" % file_inner_path)
         except Exception, err:
             self.log.debug("Error deleting file %s: %s" % (file_inner_path, err))

 def changeDb(self, auth_address, action):
     self.log.debug("Mute action %s on user %s" % (action, auth_address))
     res = self.site.content_manager.contents.db.execute(
         "SELECT * FROM content LEFT JOIN site USING (site_id) WHERE inner_path LIKE :inner_path",
         {"inner_path": "%%/%s/%%" % auth_address}
     )
     for row in res:
         site = self.server.sites.get(row["address"])
         if not site:
             continue
         dir_inner_path = helper.getDirname(row["inner_path"])
         for file_name in site.storage.walk(dir_inner_path):
             if action == "remove":
                 site.storage.onUpdated(dir_inner_path + file_name, False)
             else:
                 site.storage.onUpdated(dir_inner_path + file_name)
             site.onFileDone(dir_inner_path + file_name)

 def getDbFiles(self):
     for content_inner_path, content in self.site.content_manager.contents.iteritems():
         # content.json file itself
         if self.isFile(content_inner_path):
             yield content_inner_path, self.getPath(content_inner_path)
         else:
             self.log.error("[MISSING] %s" % content_inner_path)
         # Data files in content.json
         content_inner_path_dir = helper.getDirname(content_inner_path)  # Content.json dir relative to site
         for file_relative_path in content.get("files", {}).keys() + content.get("files_optional", {}).keys():
             if not file_relative_path.endswith(".json"):
                 continue  # We only interesed in json files
             file_inner_path = content_inner_path_dir + file_relative_path  # File Relative to site dir
             file_inner_path = file_inner_path.strip("/")  # Strip leading /
             if self.isFile(file_inner_path):
                 yield file_inner_path, self.getPath(file_inner_path)
             else:
                 self.log.error("[MISSING] %s" % file_inner_path)