Storm [practice series-how to write a crawler-encapsulation of Protocol]

Description: encapsulation of Protocol

 
Package COM. digitalpebble. storm. crawler. fetcher; import COM. digitalpebble. storm. crawler. util. configuration; public interface protocol {public protocolresponse getprotocoloutput (string URL) throws exception; Public void configure (configuration conf );}
 

Encapsulation of protoclfactory

Package COM. digitalpebble. storm. crawler. fetcher; import java.net. URL; import Java. util. weakhashmap; import COM. digitalpebble. storm. crawler. fetcher. asynchttpclient. ahprotocol; import COM. digitalpebble. storm. crawler. util. configuration;/*** @ author Yin Shuai **/public class protocolfactory {private final configuration config; private final weakhashmap <string, protocol> cache = new weakhashmap <string, P Rotocol> (); Public protocolfactory (configuration conf) {Config = conf ;} /** returns an instance of the protocol to use for a given URL **/Public synchronized protocol getprotocol (URL) {// get the protocol string protocol = URL. getprotocol (); Protocol pp = cache. get (Protocol); If (PP! = NULL) return pp; // Yuk! Hardcoded for now pp = new ahprotocol (); pp. Configure (config); cache. Put (protocol, pp); Return pp ;}}

Encapsulation of protocolresponse

Package COM. digitalpebble. storm. crawler. fetcher; import Java. util. hashmap; public class protocolresponse {final byte [] content; Final int statuscode; Final hashmap <string, string []> metadata; Public protocolresponse (byte [] C, int s, hashmap <string, string []> MD) {content = C; statuscode = s; Metadata = md;} public byte [] getcontent () {return content ;} public int getstatuscode () {return statuscode;} public hashmap <string, string []> getmetadata () {return metadata ;}}