垂直搜索引擎三》爬虫机器人模块的设计与实现

public classMyFrontierScheduler extends FrontierScheduler

{

    private static final long serialVersionUID =-1074778906898000967L;

    /**

     * @param nameName of this filter.

     */

    publicMyFrontierScheduler(String name) {

       super(name);

    }

   @Override

    protected voidschedule(CandidateURI caUri) {

    if(caUri.toString().contains("news")){

         System.out.println(caUri.toString());

         getController().getFrontier().schedule(caUri);

    }

   }

}

public class MyExtractor extends Extractor{

private String HERF ="<a(.*)href\\s*=\\s*(\"([^\"]*)\"|([^\\s>]*))(.*)>";

  private String sinaUrl ="http://(.*)news.sina.com.cn(.*).shtml";

  public MyExtractor(Stringname, String description) {

       super(name,description);

  }

public MyExtractor(String name) {

       super(name, "sinaextrator");

  }

  private static final longserialVersionUID = -963034874191929396L;

  @Override

  protected voidextract(CrawlURI curi) {

       String url="";

       try {

       HttpRecorder hr =curi.getHttpRecorder();

       if(null == hr){

             throw newException("httprecorder is null");

       }

       ReplayCharSequence  rc = hr.getReplayCharSequence();

       if(null == rc){

             return ;

       }

       String context =rc.toString();

       Pattern pattern =Pattern.compile(HERF,Pattern.CASE_INSENSITIVE);

       Matcher matcher =pattern.matcher(context);

       while(matcher.find()){

             url =matcher.group(2);

             url =url.replace("\"", "");

             //System.out.println(url);

             if(url.matches(sinaUrl)){

                  System.out.println(url);

                  <span style="color:#ff0000;">curi.createAndAddLinkRelativeToBase(url, context,Link.NAVLINK_HOP);</span>

             }

       }

       } catch (Exception e) {

             e.printStackTrace();

       }

  }

}

// ELF Hash Function

unsigned int ELFHash(char *str)

{

int hash = 0;

int x = 0;

while (*str)

{

hash = (hash << 4) + (*str++);//hash左移4位，把当前字符ASCII存入hash低四位。

if ((x = hash & 0xF0000000L) != 0)

{

//该处理，如果最高位为0，就会仅仅影响5-8位，否则会影响5-31位，因为C语言使用的算数移位

//因为1-4位刚刚存储了新加入到字符，所以不能>>28

hash ^= (x >> 24);

//上面这行代码并不会对X有影响，本身X和hash的高4位相同，下面这行代码&~即对28-31(高4位)位清零。

hash &= ~x;

}

}

//返回一个符号位为0的数，即丢弃最高位，以免函数外产生影响。(我们可以考虑，如果只有字符，符号位不可能为负)

return (hash & 0x7FFFFFFF);

}