webmagic

package com.yoyo.clourmovice;

import com.yoyo.clourmovice.pipeline.BugPagePipeLine;
import com.yoyo.clourmovice.pipeline.DownloadUriPipeLine;
import com.yoyo.clourmovice.processor.BugPageProcessor;
import com.yoyo.clourmovice.processor.DownloadUriProcessor;
import org.apache.commons.io.FileUtils;
import us.codecraft.webmagic.Spider;

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;

public class ClourMoviceApplication{

    static String domain = "http://www.mmbabe.com/";


    private static void loadingPageUrl() {
        int pageSize = 1138;
        for (int index = 1; index <= pageSize; index++) {
            Spider.create(new BugPageProcessor())
                    .addUrl("http://www.mmbabe.com/forumdisplay.php?fid=41&page=" + index)
                    .addPipeline(new BugPagePipeLine())
                    .thread(5)
                    .run();
        }
    }


    public static void loadingDownloadURI() throws IOException {
        File file = FileUtils.getFile("list.txt");
        List<String> result = FileUtils.readLines(file, Charset.forName("utf-8"));
        result.forEach(uri -> {
            String newUri = domain + uri;
            System.out.println(newUri);
            Spider.create(new DownloadUriProcessor())
                    .addUrl(newUri)
                    .addPipeline(new DownloadUriPipeLine())
                    .thread(1)
                    .run();
            return;
        });
    }



    public static void main(String[] args) throws IOException {
        loadingPageUrl();
        System.out.println("-----------------download page success");
        loadingDownloadURI();
        System.out.println("-----------------download uri success");

    }

}

 

package com.yoyo.clourmovice.pipeline;

import org.apache.commons.io.FileUtils;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;

/**
 * @Author
 */
public class BugPagePipeLine implements Pipeline {

    public static File urlFile = FileUtils.getFile("list.txt");

    @Override
    public void process(ResultItems resultitems, Task task) {
        Map<String, Object> mapResults = resultitems.getAll();
        List<String> uris = (List<String>) mapResults.get("url");
        uris.forEach(uri -> {
            try {
                FileUtils.write(urlFile, uri + "\r\n", "utf-8", true);
            } catch (IOException e) {
                e.printStackTrace();
            }
        });
    }

}

 

 

package com.yoyo.clourmovice.pipeline;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;

/**
 * @Author
 */
public class DownloadUriPipeLine implements Pipeline {

    public static File urlFile = FileUtils.getFile("uri.txt");


    @Override
    public void process(ResultItems resultItems, Task task) {
        Map<String, Object> mapResults = resultItems.getAll();
        List<String> uris = (List<String>) mapResults.get("url");
        if (CollectionUtils.isNotEmpty(uris)) {
            uris.forEach(uri -> {
                try {
                    FileUtils.write(urlFile, uri + "\r\n\n", "utf-8", true);
                } catch (IOException e) {
                    e.printStackTrace();
                }
            });
        }
    }
}

 

package com.yoyo.clourmovice.processor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

/**
 * @Author
 */
public class BugPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(100)
            .setDomain("www.mmbabe.com");

    @Override
    public void process(Page page) {
        Selectable selectable = page.getHtml().xpath("/html/body/center/form/div/div/table/tbody/tr/td[3]/a[1]/@href");
        List<String> result = selectable.all();
        page.putField("url", result);
    }

    @Override
    public Site getSite() {
        site.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
        site.addHeader("Accept-Encoding", "gzip, deflate");
        site.addHeader("Accept-Language", "zh-CN,zh;q=0.9,zh-TW;q=0.8");
        site.addHeader("Cookie", "keep-alive");
        site.addHeader("Host", "www.mmbabe.com");
        site.addHeader("Upgrade-Insecure-Requests", "1");
        site.addHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36");
        site.addHeader("Cookie", "is_use_cookied=yes; is_use_cookiex=yes; cdb_cookietime=2592000; cdb_auth=seDLoqsSbCm%2Fmn%2FrjUbqSXTmIdwNBIcSgmvTCH8sUwMN1QywiXMDlZvFyNbJfTqwUA; cdb_sid=pV3tmY");
        return site;
    }
}

 

package com.yoyo.clourmovice.processor;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.helper.StringUtil;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.ArrayList;
import java.util.List;

/**
 * @Author
 */
public class DownloadUriProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(100)
            .setDomain("www.mmbabe.com");

    @Override
    public void process(Page page) {
        Selectable selectable = page.getHtml().xpath("/html/body/center/div[4]/form/div/table/tbody/tr/td[2]/table/tbody/tr[2]/td/div[2]/text()");
        List<String> result = selectable.all();
        List<String> newResult = new ArrayList<>();
        for (String uri : result) {
            if (StringUtils.isNotBlank(uri)) {
                String[] uris = uri.split(" ");
                if (uris.length > 0) {
                    for (int index =0 ; index < uris.length; index++) {
                        String temp = uris[index];
                        if (temp.length() > 7 && temp.indexOf("thunder") >= 0) {
                            temp = temp.substring(temp.indexOf("thunder"));
                            newResult.add(temp);
                        }
                    }
                }
            }
        }

        if (CollectionUtils.isNotEmpty(newResult)) {
            System.out.println("\t" + newResult);
            page.putField("url", newResult);
        }
    }

    @Override
    public Site getSite() {
        site.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3");
        site.addHeader("Accept-Encoding", "gzip, deflate");
        site.addHeader("Accept-Language", "zh-CN,zh;q=0.9,zh-TW;q=0.8");
        site.addHeader("Cookie", "keep-alive");
        site.addHeader("Host", "www.mmbabe.com");
        site.addHeader("Upgrade-Insecure-Requests", "1");
        site.addHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36");
        site.addHeader("Cookie", "is_use_cookied=yes; is_use_cookiex=yes; cdb_cookietime=2592000; cdb_auth=seDLoqsSbCm%2Fmn%2FrjUbqSXTmIdwNBIcSgmvTCH8sUwMN1QywiXMDlZvFyNbJfTqwUA; cdb_sid=pV3tmY");
        return site;
    }
}