Multispider作为程序的入口，模仿webmagic的入口方式

Author: jikyll

src/main/java/MultiSpider.java

@@ -1,14 +1,43 @@
 
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
-import org.apache.http.impl.client.CloseableHttpClient;
+import scheduler.DispatchScheduler;
+import downloader.HttpClientDownloader;
 
-public class MultiSpider implements Runnable {
-	private  CloseableHttpClient httpClient;
-	public MultiSpider(CloseableHttpClient httpClient) {
-		this.httpClient = httpClient;
+public class MultiSpider {
+
+	private int threadNum = 1;
+	private ExecutorService service;
+	private String username;
+	private String password;
+	private DispatchScheduler scheduler;
+	
+	public MultiSpider create(String username, String password) {
+		this.username = username;
+		this.password = password;
+		HttpClientDownloader.init(username, password);
+		scheduler = new DispatchScheduler();
+		scheduler.startProcess();
+		return new MultiSpider();
 	}
+
 	public void run() {
+		service = Executors.newFixedThreadPool(this.threadNum);
+		Future future = service.submit(new Callable() {
+
+			public Object call() throws Exception {
+				scheduler.taskProcceed();
+				return null;
+			}
 
+		});
 	}
 
+	public MultiSpider thread(int num) {
+		this.threadNum = num;
+		return this;
+	}
 }

src/main/java/container/URLContainer.java

@@ -19,24 +19,25 @@ public interface Downloader {
 	/**
 	 * 爬取题目列表
 	 * @param url
+	 * @return 
 	 */
-	public void problemListDownloader(String url);
+	public List<String> problemListDownloader();
 
 	/**
 	 * 进入题目描述页面，爬取题目提交url列表
 	 * @param url
 	 */
-	public void problemDescriptionDownloader(String url);
+	public List<String> problemDescriptionDownloader(String url);
 
 	/**
 	 *	进入题目提交列表页面，爬取code所在页面url 
 	 * @param url
 	 */
-	public void submissionListDownloader(String url, String name);
+	public List<String> submissionListDownloader(String url, String name);
 
 	/**
 	 * 进入code所在页面，爬取code
 	 * @param url
 	 */
-	public void codePageDownloader(String url);
+	public List<String> codePageDownloader(String url);
 }

src/main/java/downloader/Downloader.java

@@ -11,7 +11,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.concurrent.BlockingQueue;
 
 import org.apache.http.Header;
 import org.apache.http.HeaderIterator;
@@ -29,10 +28,10 @@
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
 
-import container.URLContainer;
 import us.codecraft.webmagic.selector.XpathSelector;
 
-public class HttpClientDownloader extends URLContainer  implements Downloader {
+public class HttpClientDownloader implements Downloader {
+	private static final String ALGORITHMS = "https://www.leetcode.com/problemset/algorithms/";
 	private static final String INDEX_URL = "https://leetcode.com";                                    														//首页的url地址
 	private static final String LOGIN_URL = "https://leetcode.com/accounts/login/";																//登录页面url地址			
 	private static final String PASSWORD = "thd04180015";																										//登录密码
@@ -51,16 +50,12 @@ public class HttpClientDownloader extends URLContainer  implements Downloader {
 	private static final String problemNamePath = "//div[@class='col-md-12']/h4/a/text()";                                                  //题目名称Xpath路径
 	private static final String codePath = "//div[@class='ace_content']/text()";																			//得到代码Xpath路径
 	private static CloseableHttpClient httpClient;
-	/**
-	 * 
-	 * 创建一个新的实例 HttpClientDownloader.
-	 *
-	 */
-	public HttpClientDownloader() {
-		init();
-	}
+	private static final HttpClientDownloader downloader = new HttpClientDownloader();
 
-	public void init() {
+	public static HttpClientDownloader getInstance() {
+		return downloader;
+	}
+	public static void init(String username, String password) {
 		httpClient = HttpClients.createDefault();
 		HttpGet httpGet = new HttpGet(INDEX_URL);
 		CloseableHttpResponse response1;
@@ -87,8 +82,8 @@ public void init() {
 		httpPost.addHeader("Referer", LOGIN_URL);
 		httpPost.addHeader("Origin", INDEX_URL);
 		Map<String, String> map = new HashMap<String, String>();
-		map.put("login", USER_NAME);
-		map.put("password", PASSWORD);
+		map.put("login", username);
+		map.put("password", password);
 		map.put("csrfmiddlewaretoken", cookieString);
 		map.put("remember", "on");
 		try {
@@ -196,20 +191,21 @@ public static String getCookie(HttpResponse httpResponse) {
 		return null;
 	}
 
-	public void problemListDownloader(String url) {
-		doDispatcher(url, problemQueue);
+	public List<String> problemListDownloader() {
+		return doDispatcher(ALGORITHMS);
 	}
 
-	public void problemDescriptionDownloader(String url) {
-		doDispatcher(url, problemSubmission);
+	public List<String> problemDescriptionDownloader(String url) {
+		return doDispatcher(url);
 	}
 
-	public void submissionListDownloader(String url, String name) {
-		doDispatcher(url, problemCodePage, name);
+	public List<String> submissionListDownloader(String url, String name) {
+		return doDispatcher(url, name, httpClient);
 	}
 
-	public void codePageDownloader(String url) {
+	public List<String> codePageDownloader(String url) {
 		//爬取代码
+		return null;
 	}
 
 	public String getHtml(HttpResponse response) {
@@ -247,26 +243,23 @@ public String getHtml(HttpResponse response) {
 	 * @exception
 	 * @since  1.0.0
 	 */
-	public void doDispatcher(String url, BlockingQueue<String> blockingQueue) {
+	public List<String> doDispatcher(String url) {
 		HttpGet httpGet1 = new HttpGet(url);
 		try {
 			HttpResponse response1 = httpClient.execute(httpGet1);
 			printResponse(response1);
 
 			XpathSelector xpathSelector = new XpathSelector(problemLinkPath);
 			List<String> listTmp = xpathSelector.selectList(getHtml(response1));
-			for (String string: listTmp) {
-				blockingQueue.put(string);
-			}
 			HttpEntity entity1 = response1.getEntity();
 			EntityUtils.consume(entity1);
+			return listTmp;
 		} catch (ClientProtocolException e) {
 			System.out.println(e.getMessage());
 		} catch (IOException e) {
 			System.out.println(e.getMessage());
-		} catch (InterruptedException e) {
-			System.out.println(e.getMessage());
 		}
+		return null;
 	}
 	/**
 	 * 
@@ -279,25 +272,22 @@ public void doDispatcher(String url, BlockingQueue<String> blockingQueue) {
 	 * @exception
 	 * @since  1.0.0
 	 */
-	public void doDispatcher(String url, BlockingQueue<String> blockingQueue, String name) {
+	public List<String> doDispatcher(String url, String name, CloseableHttpClient httpClient) {
 		HttpGet httpGet1 = new HttpGet(url);
 		try {
 			HttpResponse response1 = httpClient.execute(httpGet1);
 			printResponse(response1);
 
 			XpathSelector xpathSelector = new XpathSelector(problemLinkPath);
 			List<String> listTmp = xpathSelector.selectList(getHtml(response1));
-			for (String string: listTmp) {
-				blockingQueue.put(string);
-			}
 			HttpEntity entity1 = response1.getEntity();
 			EntityUtils.consume(entity1);
+			return listTmp;
 		} catch (ClientProtocolException e) {
 			System.out.println(e.getMessage());
 		} catch (IOException e) {
 			System.out.println(e.getMessage());
-		} catch (InterruptedException e) {
-			System.out.println(e.getMessage());
 		}
+		return null;
 	}
 }