import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; //导入依赖的package包/类
@Test
public void test() throws Exception {
    Spider.create(new MySqlPageProcessor(htmlService))
            .scheduler(new FileCacheQueueScheduler("F:\\webmagic\\cache\\"))
            .addUrl("http://www.cnblogs.com/")
            .addPipeline(new MySqlPipeline(htmlService))
            .thread(10)
            .run();
}
 

import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; //导入依赖的package包/类
public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
    Spider.create(new MamacnPageProcessor())
            .setScheduler(new FileCacheQueueScheduler("/data/webmagic/mamacn"))
            .addUrl("http://www.mama.cn/photo/t1-p1.html")
            .addPipeline(new OneFilePipeline("/data/webmagic/mamacn/data"))
            .thread(5)
            .run();
}
 

import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; //导入依赖的package包/类
public static void main(String[] args) {

        Spider.create(new GithubRepoPageProcessor())
                //从"https://github.com/code4craft"开始抓
                .addUrl("https://github.com/code4craft")
                .scheduler(new FileCacheQueueScheduler("K:\\data\\webmagic"))
                .addPipeline(new FilePipeline())
                //开启5个线程抓取
                .thread(5)
                //启动爬虫
                .run();
    }
 

import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; //导入依赖的package包/类
@Ignore
@Test
public void test() throws IOException {
    DiaoyuwengProcessor diaoyuwengProcessor = new DiaoyuwengProcessor();
    JsonFilePipeline pipeline = new JsonFilePipeline("/data/webmagic/");
    Spider.create(diaoyuwengProcessor).pipeline(new FilePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
            run();
}
 

import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; //导入依赖的package包/类
@Ignore
@Test
public void test() throws IOException {
    SinaBlogProcesser sinaBlogProcesser = new SinaBlogProcesser();
    //pipeline是抓取结束后的处理
    //默认放到/data/webmagic/ftl/[domain]目录下
    JsonFilePipeline pipeline = new JsonFilePipeline("/data/webmagic/");
    //Spider.me()是简化写法，其实就是new一个啦
    //Spider.pipeline()设定一个pipeline，支持链式调用
    //ConsolePipeline输出结果到控制台
    //FileCacheQueueSchedular保存url，支持断点续传，临时文件输出到/data/temp/webmagic/cache目录
    //Spider.run()执行
    Spider.create(sinaBlogProcesser).pipeline(new FilePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
            run();
}
 

import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; //导入依赖的package包/类
@Ignore
    @Test
    public void testGlobalSpider(){
//        PageProcessor pageProcessor = new MeicanProcessor();
//        Spider.me().pipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
//                processor(pageProcessor).run();
        SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space", "http://www.diaoyuweng.com/thread-*-1-1.html");
        System.out.println(pageProcessor2.getSite().getCharset());
        pageProcessor2.getSite().setSleepTime(500);
        Spider.create(pageProcessor2).pipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
                run();


    }
 

import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; //导入依赖的package包/类
@Ignore
@Test
public void test() throws IOException {
    SinaBlogProcessor sinaBlogProcessor = new SinaBlogProcessor();
    //pipeline是抓取结束后的处理
    //默认放到/data/webmagic/ftl/[domain]目录下
    JsonFilePipeline pipeline = new JsonFilePipeline("/data/webmagic/");
    //Spider.me()是简化写法，其实就是new一个啦
    //Spider.pipeline()设定一个pipeline，支持链式调用
    //ConsolePipeline输出结果到控制台
    //FileCacheQueueSchedular保存url，支持断点续传，临时文件输出到/data/temp/webmagic/cache目录
    //Spider.run()执行
    Spider.create(sinaBlogProcessor).pipeline(new FilePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
            run();
}
 

import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; //导入依赖的package包/类
@Ignore
    @Test
    public void testGlobalSpider(){
//        PageProcessor pageProcessor = new MeicanProcessor();
//        Spider.me().pipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
//                processor(pageProcessor).run();
        SimplePageProcessor pageProcessor2 = new SimplePageProcessor( "http://www.diaoyuweng.com/thread-*-1-1.html");
        System.out.println(pageProcessor2.getSite().getCharset());
        pageProcessor2.getSite().setSleepTime(500);
        Spider.create(pageProcessor2).addUrl("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space").addPipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
                run();


    }
 

import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; //导入依赖的package包/类
public static void main(String[] args) {
    OOSpider.create(Site.me().addStartUrl("https://github.com/explore").setSleepTime(0).setRetryTimes(3),
            new JsonFilePageModelPipeline(), GithubRepo.class)
            .scheduler(new FileCacheQueueScheduler("/data/webmagic/cache/")).thread(15).run();
}
 

import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; //导入依赖的package包/类
public static void main(String[] args) {
    OOSpider.create(Site.me().setSleepTime(0).setRetryTimes(3),
            new JsonFilePageModelPipeline(), GithubRepo.class)
            .addUrl("https://github.com/explore")
            .setScheduler(new FileCacheQueueScheduler("/data/webmagic/cache/")).thread(15).run();
}