import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
@Override
public void process(Page page) {
    List<Selectable> nodes = page.getHtml().xpath("//ul[@id=ma-thumb-list]/li").nodes();
    StringBuilder accum = new StringBuilder();
    for (Selectable node : nodes) {
        accum.append("img:").append(node.xpath("//a/@href").get()).append("\n");
        accum.append("title:").append(node.xpath("//img/@alt").get()).append("\n");
    }
    page.putField("",accum.toString());
    if (accum.length() == 0) {
        page.setSkip(true);
    }
    page.addTargetRequests(page.getHtml().links().regex("http://www\\.mama\\.cn/photo/.*\\.html").all());
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
@Override
public void process(Page page) {
    Html html = page.getHtml();
    count++;
    Selectable sel = html.links().regex("("+URLREG+"[a-zA-Z0-9\\d]+.shtml)");
    page.addTargetRequests(sel.all());
    if(count!=1) {
        for(String f:fields){
            String key = config.getProperty(f);
            if(!"".equals(key)) {
                String pro = config.getProperty(f+"_deleteTag");
                if(pro!=null) {
                    String[] tags = pro.split(",");
                    String value = html.xpath(key).toString();
                    if("content".equals(f))
                        page.putField(f,value);
                    else
                        page.putField(f, deleteTag(value, tags).replaceAll(config.getProperty(f + "_replaceStr"), ""));
                }
            }
        }
        try {
            page.putField("area", new String(Area.getBytes("ISO-8859-1"),"utf-8"));
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
    }
    System.out.println("############=====> "+count);
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
@Test
public void testNthNodesGet(){
	Html html = new Html("<a data-tip=\"p$t$xxx\" href=\"/xx/xx\">xx</a>");
	assertThat(html.xpath("//a[1]/@href").get()).isEqualTo("/xx/xx");
	Selectable selectable = html.xpath("//a[1]").nodes().get(0);
	assertThat(selectable.xpath("/a/@href").get()).isEqualTo("/xx/xx");
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
@Override
public Selectable getUrl() {
    return page.getUrl();
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
public void parseAllTopics(Page page) {
	Selectable selectable = page.getHtml().xpath("//div[@id='group-topics']//a");
	List<Selectable> selectables = selectable.nodes();
	selectables.stream().filter((s) -> s.match()).forEach(System.out::print);
	page.putField("allTopics", selectables);
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
public void setHtml(Selectable html) {
    this.html = html;
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
public void setUrl(Selectable url) {
    this.url = url;
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
/**
 * 获取页面的html内容
 *
 * @return html 页面的html内容
 */
public Selectable getHtml() {
    return html;
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
/**
 * 获取页面的Url
 *
 * @return url 当前页面的url，可用于抽取
 */
public Selectable getUrl() {
    return url;
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
/**
 * 设置url
 *
 * @param url
 */
public void setUrl(Selectable url) {
    this.url = url;
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
/**
 * get url of current page
 *
 * @return url of current page
 */
public Selectable getUrl() {
    return url;
}
 

import us.codecraft.webmagic.selector.Selectable; //导入依赖的package包/类
Selectable getUrl();