def test_resolver_dir_wildcard(self):
        resolver = DefaultResolverMixin.make_best_resolver(os.path.join(get_test_dir(), '*', ''))

        cdx = CDXObject()
        res = resolver('example.warc.gz', cdx)
        assert len(res) == 1
        assert res[0] == os.path.join(get_test_dir(), 'warcs', 'example.warc.gz')

    def test_resolver_dir_wildcard_as_file_url(self):
        url = to_file_url(get_test_dir()) +  '/*/'
        resolver = DefaultResolverMixin.make_best_resolver(url)

        cdx = CDXObject()
        res = resolver('example.warc.gz', cdx)
        assert len(res) == 1
        assert res[0] == os.path.abspath(os.path.join(get_test_dir(), 'warcs', 'example.warc.gz'))

    def setup_class(cls):
        super(TestZipnumAutoDir, cls).setup_class('config_test.yaml')

        manager(['init', 'testzip'])

        cls.archive_dir = os.path.join(cls.root_dir, '_test_colls', 'testzip', 'archive')
        cls.index_dir = os.path.join(cls.root_dir, '_test_colls', 'testzip', 'indexes')

        zip_cdx = os.path.join(get_test_dir(), 'zipcdx')

        shutil.copy(os.path.join(zip_cdx, 'zipnum-sample.idx'), cls.index_dir)
        shutil.copy(os.path.join(zip_cdx, 'zipnum-sample.cdx.gz'), cls.index_dir)
        shutil.copy(os.path.join(zip_cdx, 'zipnum-sample.loc'), cls.index_dir)

        shutil.copy(os.path.join(get_test_dir(), 'warcs', 'iana.warc.gz'), cls.archive_dir)

def test_cdxj_resolve_revisit_2():
    # Resolve Revisit -- cdxj minimal -- output also json
    results = cdx_ops_test_data(
        url="http://example.com/?example=1",
        sources=[get_test_dir() + "cdxj/example-no-digest.cdxj"],
        resolveRevisits=True,
    )
    assert len(results) == 2
    assert dict(results[0]) == {
        "urlkey": "com,example)/?example=1",
        "timestamp": "20140103030321",
        "url": "http://example.com?example=1",
        "length": "1043",
        "filename": "example.warc.gz",
        "offset": "333",
        "orig.length": "-",
        "orig.offset": "-",
        "orig.filename": "-",
    }

    assert dict(results[1]) == {
        "urlkey": "com,example)/?example=1",
        "timestamp": "20140103030341",
        "url": "http://example.com?example=1",
        "length": "553",
        "filename": "example.warc.gz",
        "mime": "warc/revisit",
        "offset": "1864",
        "orig.length": "-",
        "orig.offset": "-",
        "orig.filename": "-",
    }

def test_zip_prefix_load():

    tmpdir = tempfile.mkdtemp()
    try:
        shutil.copy(test_zipnum, tmpdir)
        shutil.copy(get_test_dir() + 'zipcdx/zipnum-sample.cdx.gz',
                    os.path.join(tmpdir, 'zipnum'))

        config={}
        config['shard_index_loc'] = dict(match='(.*)',
                                         replace=r'\1')
        server = CDXServer(os.path.join(tmpdir, 'zipnum-sample.idx'),
                           config=config)


        # Test Page Count
        results = server.load_cdx(url='iana.org/',
                                  matchType='domain',
                                  showNumPages=True)

        results = list(results)
        assert len(results) == 1, results
        assert json.loads(results[0]) == {"blocks": 38, "pages": 4, "pageSize": 10}


        # Test simple query
        results = server.load_cdx(url='iana.org/')
        results = list(results)
        assert len(results) ==3, results
        assert '20140126200624' in results[0]
        assert '20140127171238' in results[1]
        assert 'warc/revisit' in results[2]

    finally:
        shutil.rmtree(tmpdir)

    def test_dash_default_max(self):
        headers = {'Content-Type': 'application/dash+xml'}
        with open(os.path.join(get_test_dir(), 'text_content', 'sample_dash.mpd'), 'rt') as fh:
            content = fh.read()

        headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701oe_',
                                                  url='http://example.com/path/manifest.mpd')

        assert headers.headers == [('Content-Type', 'application/dash+xml')]

        filtered = """\
<?xml version='1.0' encoding='UTF-8'?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" mediaPresentationDuration="PT0H3M1.63S" minBufferTime="PT1.5S" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" type="static">
  <Period duration="PT0H3M1.63S" start="PT0S">
    <AdaptationSet>
      <ContentComponent contentType="video" id="1" />
      <Representation bandwidth="4190760" codecs="avc1.640028" height="1080" id="1" mimeType="video/mp4" width="1920">
        <BaseURL>http://example.com/video-10.mp4</BaseURL>
        <SegmentBase indexRange="674-1149">
          <Initialization range="0-673" />
        </SegmentBase>
      </Representation>
      </AdaptationSet>
    <AdaptationSet>
      <ContentComponent contentType="audio" id="2" />
      <Representation bandwidth="255236" codecs="mp4a.40.2" id="7" mimeType="audio/mp4" numChannels="2" sampleRate="44100">
        <BaseURL>http://example.com/audio-2.mp4</BaseURL>
        <SegmentBase indexRange="592-851">
          <Initialization range="0-591" />
        </SegmentBase>
      </Representation>
      </AdaptationSet>
  </Period>
</MPD>"""
        assert b''.join(gen).decode('utf-8') == filtered

    def test_path_index_resolvers(self):
        path = os.path.join(get_test_dir(), 'text_content', 'pathindex.txt')
        path_index = PathIndexResolver(path)

        cdx = CDXObject()
        assert list(path_index('example.warc.gz', cdx)) == ['invalid_path', 'sample_archive/warcs/example.warc.gz']
        assert list(path_index('iana.warc.gz', cdx)) == ['sample_archive/warcs/iana.warc.gz']
        assert list(path_index('not-found.gz', cdx)) == []

def test_local_no_head_only_title():
    status_headers, buff = get_rewritten(get_test_dir() + 'text_content/sample_no_head_2.html',
                                         urlrewriter,
                                         head_insert_func,
                                         'com,example,test)/')

    # wombat insert added
    assert '<script src="/static/__pywb/wombat.js"> </script>' in buff

    def setup_class(cls):
        super(MementoOverrideTests, cls).setup_class()

        # Load expected link headers
        MementoOverrideTests.link_header_data = None
        with open(to_path(get_test_dir() + '/text_content/link_headers.yaml')) as fh:
            MementoOverrideTests.link_header_data = yaml.load(fh)

        MementoOverrideTests.orig_get_timegate_links = MementoIndexSource.get_timegate_links

def test_local_unclosed_script():
    status_headers, buff = get_rewritten(get_test_dir() + 'text_content/sample_unclosed_script.html',
                                         urlrewriter,
                                         head_insert_func,
                                         'com,example,test)/')

    # wombat insert added
    assert '<head><script src="/static/__pywb/wombat.js"> </script>' in buff, buff

    # JS location and JS link rewritten
    assert 'window.WB_wombat_location = "/pywb/20131226101010/http:\/\/example.com/dynamic_page.html";\n}\n</script>' in buff, buff

def test_local_no_head_banner_only():
    status_headers, buff = get_rewritten(get_test_dir() + 'text_content/sample_no_head.html',
                                         bn_urlrewriter,
                                         head_insert_func,
                                         'com,example,test)/')

    # wombat insert added
    assert '<script src="/static/__pywb/wombat.js"> </script>' in buff

    # location NOT rewritten
    assert 'window.location = "/other.html"' in buff

    # link NOT rewritten
    assert '"another.html"' in buff

def test_local_banner_only_no_rewrite():
    status_headers, buff = get_rewritten(get_test_dir() + 'text_content/sample.html',
                                         bn_urlrewriter,
                                         head_insert_func,
                                         'com,example,test)/')

    # wombat insert added
    assert '<head><script src="/static/__pywb/wombat.js"> </script>' in buff

    # JS location NOT rewritten, JS link NOT rewritten
    assert 'window.location = "http:\/\/example.com/dynamic_page.html"' in buff, buff

    # link NOT rewritten
    assert '"another.html"' in buff

def test_local_2_no_rewrite():
    status_headers, buff = get_rewritten(get_test_dir() + 'text_content/sample.html',
                                         urlrewriter,
                                         head_insert_func,
                                         'example,example,test,norewrite)/')

    # wombat insert added
    assert '<script src="/static/__pywb/wombat.js"> </script>' in buff

    # JS location NOT rewritten, JS link NOT rewritten
    assert 'window.location = "http:\/\/example.com/dynamic_page.html"' in buff

    # still link rewrite in HTML
    assert '"/pywb/20131226101010/http://example.com/some/path/another.html"' in buff

def test_local_1():
    status_headers, buff = get_rewritten(get_test_dir() + 'text_content/sample.html',
                                         urlrewriter,
                                         head_insert_func,
                                         'example,example,test,all)/')

    # wombat insert added
    assert '<head><script src="/static/__pywb/wombat.js"> </script>' in buff, buff

    # JS location and JS link rewritten
    assert 'window.WB_wombat_location = "/pywb/20131226101010/http:\/\/example.com/dynamic_page.html"' in buff

    # link rewritten
    assert '"/pywb/20131226101010/http://example.com/some/path/another.html"' in buff

def test_local_no_head():
    status_headers, buff = get_rewritten(get_test_dir() + 'text_content/sample_no_head.html',
                                         urlrewriter,
                                         head_insert_func,
                                         'com,example,test)/')

    # wombat insert added
    assert '<script src="/static/__pywb/wombat.js"> </script>' in buff, buff

    # location rewritten
    assert 'window.WB_wombat_location = "/other.html"' in buff, buff

    # link rewritten
    assert '"/pywb/20131226101010/http://example.com/some/path/another.html"' in buff, buff

def test_local_2_link_only_rewrite():
    status_headers, buff = get_rewritten(get_test_dir() + 'text_content/sample.html',
                                         urlrewriter,
                                         head_insert_func,
                                         'example,example,test)/nolocation_rewrite')

    # no wombat insert
    assert '<head><script src="/static/default/wombat.js"> </script>' not in buff

    # JS location NOT rewritten, JS link rewritten
    assert 'window.location = "/pywb/20131226101010/http:\/\/example.com/dynamic_page.html"' in buff

    # still link rewrite
    assert '"/pywb/20131226101010/http://example.com/some/path/another.html"' in buff

    def test_hls_default_max(self):
        headers = {'Content-Type': 'application/vnd.apple.mpegurl'}
        with open(os.path.join(get_test_dir(), 'text_content', 'sample_hls.m3u8'), 'rt') as fh:
            content = fh.read()

        headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701oe_',
                                                  url='http://example.com/path/master.m3u8')

        assert headers.headers == [('Content-Type', 'application/vnd.apple.mpegurl')]
        filtered = """\
#EXTM3U
#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="WebVTT",NAME="English",DEFAULT=YES,AUTOSELECT=YES,FORCED=NO,URI="https://example.com/subtitles/"
#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=4495000,RESOLUTION=1920x1080,CODECS="avc1.640028, mp4a.40.2",SUBTITLES="WebVTT"
http://example.com/video_6.m3u8
"""

        assert b''.join(gen).decode('utf-8') == filtered

    def test_dash_fb_in_js(self):
        headers = {'Content-Type': 'text/javascript'}
        with open(os.path.join(get_test_dir(), 'text_content', 'sample_dash.mpd'), 'rt') as fh:
            content = 'dash_manifest:"' + fh.read().encode('unicode-escape').decode('utf-8')

        rep_ids = r'\n",dash_prefetched_representation_ids:["4","5"]'
        content += rep_ids

        headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701js_',
                                                  url='http://facebook.com/example/dash/manifest.mpd')

        assert headers.headers == [('Content-Type', 'text/javascript')]

        result = b''.join(gen).decode('utf-8')

        # 4, 5 representations removed, replaced with default 1, 7
        assert 'dash_prefetched_representation_ids:["1", "7"]' in result
        assert rep_ids not in result

    def test_hls_custom_max_bandwidth(self):
        headers = {'Content-Type': 'application/x-mpegURL'}
        with open(os.path.join(get_test_dir(), 'text_content', 'sample_hls.m3u8'), 'rt') as fh:
            content = fh.read()

        metadata = {'adaptive_max_bandwidth': 2000000}

        headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701oe_',
                                                  url='http://example.com/path/master.m3u8',
                                                  warc_headers={'WARC-JSON-Metadata': json.dumps(metadata)})

        assert headers.headers == [('Content-Type', 'application/x-mpegURL')]
        filtered = """\
#EXTM3U
#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="WebVTT",NAME="English",DEFAULT=YES,AUTOSELECT=YES,FORCED=NO,URI="https://example.com/subtitles/"
#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1002000,RESOLUTION=640x360,CODECS="avc1.77.30, mp4a.40.2",SUBTITLES="WebVTT"
http://example.com/video_4.m3u8
"""

        assert b''.join(gen).decode('utf-8') == filtered

    def test_dash_custom_max_resolution(self):
        headers = {'Content-Type': 'application/dash+xml'}
        with open(os.path.join(get_test_dir(), 'text_content', 'sample_dash.mpd'), 'rt') as fh:
            content = fh.read()

        metadata = {'adaptive_max_resolution': 921600,
                    'adaptive_max_bandwidth': 2000000}

        headers, gen, is_rw = self.rewrite_record(headers, content, ts='201701oe_',
                                                  url='http://example.com/path/manifest.mpd',
                                                  warc_headers={'WARC-JSON-Metadata': json.dumps(metadata)})

        assert headers.headers == [('Content-Type', 'application/dash+xml')]

        filtered = """\
<?xml version='1.0' encoding='UTF-8'?>
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" mediaPresentationDuration="PT0H3M1.63S" minBufferTime="PT1.5S" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" type="static">
  <Period duration="PT0H3M1.63S" start="PT0S">
    <AdaptationSet>
      <ContentComponent contentType="video" id="1" />
      <Representation bandwidth="2073921" codecs="avc1.4d401f" height="720" id="2" mimeType="video/mp4" width="1280">
        <BaseURL>http://example.com/video-9.mp4</BaseURL>
        <SegmentBase indexRange="708-1183">
          <Initialization range="0-707" />
        </SegmentBase>
      </Representation>
      </AdaptationSet>
    <AdaptationSet>
      <ContentComponent contentType="audio" id="2" />
      <Representation bandwidth="255236" codecs="mp4a.40.2" id="7" mimeType="audio/mp4" numChannels="2" sampleRate="44100">
        <BaseURL>http://example.com/audio-2.mp4</BaseURL>
        <SegmentBase indexRange="592-851">
          <Initialization range="0-591" />
        </SegmentBase>
      </Representation>
      </AdaptationSet>
  </Period>
</MPD>"""

        assert b''.join(gen).decode('utf-8') == filtered