import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
protected void initComponent() {
    if (downloader == null) {
        this.downloader = new HttpClientDownloader();
    }
    if (pipelines.isEmpty()) {
        pipelines.add(new ConsolePipeline());
    }
    downloader.setThread(threadNum);
    if (threadPool == null || threadPool.isShutdown()) {
        if (executorService != null && !executorService.isShutdown()) {
            threadPool = new CountableThreadPool(threadNum, executorService);
        } else {
            threadPool = new CountableThreadPool(threadNum);
        }
    }
    if (startRequests != null) {
        for (Request request : startRequests) {
            addRequest(request);
        }
        startRequests.clear();
    }
    startTime = new Date();
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public static void main(String[] args) {
    init();
    String key = null;
    key = readKey(key);
    System.out.println("The demo started and will last 20 seconds...");
    //Start spider
    OOSpider.create(Site.me(), clazzMap.get(key))
            .addUrl(urlMap.get(key))
            .addPipeline(new MultiPagePipeline())
            .addPipeline(new ConsolePipeline())
            .runAsync();

    try {
        Thread.sleep(20000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    System.out.println("The demo stopped!");
    System.out.println("To more usage, try to customize your own Spider!");
    System.exit(0);
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public static void main(String[] args) {
    init();
    String key = null;
    key = readKey(key);
    System.out.println("The demo started and will last 20 seconds...");
    //Start spider
    OOSpider.create(Site.me().addStartUrl(urlMap.get(key)), clazzMap.get(key)).pipeline(new PagedPipeline()).pipeline(new ConsolePipeline()).runAsync();

    try {
        Thread.sleep(20000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    System.out.println("The demo stopped!");
    System.out.println("To more usage, try to customize your own Spider!");
    System.exit(0);
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public static void main(String[] args) {
    init();
    String key = null;
    key = readKey(key);
    System.out.println("The demo started and will last 20 seconds...");
    //Start spider
    OOSpider.create(Site.me(), clazzMap.get(key)).addUrl(urlMap.get(key)).addPipeline(new MultiPagePipeline()).addPipeline(new ConsolePipeline()).runAsync();

    try {
        Thread.sleep(20000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    System.out.println("The demo stopped!");
    System.out.println("To more usage, try to customize your own Spider!");
    System.exit(0);
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public ConcurrentHashMap<byte[], byte[]> start(List<String> pageNumberList, String tiebaName) {
    isAddTarget=false;
    map.clear();
    this.tiebaName=tiebaName;
    this.pageNumberList = pageNumberList;
    this.url = tieBaConfiguration.getTiebaContentPageUrl();
    Spider.create(this).addUrl(url).addPipeline(new ConsolePipeline())
            // 开启5个线程抓取
            .thread(30)
            // 启动爬虫
            .run();
    return  map;
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public ConcurrentHashMap<String, TopBean> start(String name) {
        isAddTarget=false;
        map.clear();
        tiebaName = name;
        tiebaUrl = tieba.getTiebaTopUrl() + name;
        Spider.create(this).addUrl(tiebaUrl).addPipeline(new ConsolePipeline())
                // 开启5个线程抓取
//                .thread(200)
                //服务器上面线程数不能开启多 防止链接异常
                .thread(2)
                // 启动爬虫
                .run();
        return map;
    }
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public void start(String tiebaName) {
   isAddTarget=false;
    pageNumberList.clear();
   this.tiebaName=tiebaName;
    tiebaUrl = tieBaConfiguration.getTiebaContentUrl() + tiebaName;
   endNum=Integer.parseInt(tieBaConfiguration.getTiebaContentPageEndNum());
    Spider.create(this).addUrl(tiebaUrl).addPipeline(new ConsolePipeline())
            // 开启5个线程抓取
            .thread(2)
            // 启动爬虫
            .run();
   if(pageNumberList.size()>0) {
       redisTemplate.convertAndSend(tieBaConfiguration.getTiebaContentIdTopic(), JSONObject.toJSONString(pageNumberList));
   }
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public void action() {
	Spider.create(new DoubanHouseProcessor())
			.addUrl(HouseSource.DOUBAN)
			.addPipeline(new ConsolePipeline())
			.run();

	Spider.create(new TongChengHouseProcessor())
			.addUrl(HouseSource.COM_58)
			.addPipeline(new ConsolePipeline())
			.run();
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
protected void checkComponent() {
	if (downloader == null) {
		this.downloader = new HttpClientDownloader();
	}
	if (pipelines.isEmpty()) {
		pipelines.add(new ConsolePipeline());
	}
	downloader.setThread(threadNum);
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public static void main(String[] args) {
    Spider.create(new VogueBrandSpider()).addUrl("http://brand.vogue.com.cn")
            .addPipeline(new ConsolePipeline()).run();
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public static void main(String[] args) {
    OOSpider.create(Site.me(), News163.class).addUrl("http://news.163.com/13/0802/05/958I1E330001124J_2.html")
            .scheduler(new RedisScheduler("localhost")).addPipeline(new MultiPagePipeline()).addPipeline(new ConsolePipeline()).run();
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public static void main(String[] args) {
    OOSpider.create(Site.me().addStartUrl("http://news.163.com/13/0802/05/958I1E330001124J_2.html"), News163.class)
            .scheduler(new RedisScheduler("localhost")).clearPipeline().pipeline(new PagedPipeline()).pipeline(new ConsolePipeline()).run();
}
 

import us.codecraft.webmagic.pipeline.ConsolePipeline; //导入依赖的package包/类
public static void main(String[] args) {
    Spider.create(new OschinaBlogPageProcesser()).pipeline(new ConsolePipeline()).run();
}