项目结构

├─conf
│      application.conf
│
├─src
│  ├─main
│  │  ├─java
│  │  │  └─com
│  │  │      └─aa
│               └─httpClient
│                    AppConfig.java
│                    Util.java
│                    ZhiHu.java
├─ pom.xml

maven依赖

    <properties >
        <config.version>1.3.0</config.version>
    </properties>

    <dependencies>

        <!--config-->
        <dependency>
            <groupId>com.typesafe</groupId>
            <artifactId>config</artifactId>
            <version>${config.version}</version>
        </dependency>

        <!--jsoup-->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.11.3</version>
        </dependency>

        <!--httpClient-->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.2</version>
        </dependency>

        <!--ioUtils-->
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.4</version>
        </dependency>

配置文件

config = {
    url = "https://www.zhihu.com/question/325648700"
    savePath = "E:\\resourse\\image\\wallpaper\\ZhiHu"
    }

Config类

public class AppConfig  {
    private static String CONFIG_PATH =new File("").getAbsolutePath()+"\\conf\\application.conf";
    private static Config config;

    private AppConfig(){}

    public synchronized static Config getInstance(){
        File file = new File(CONFIG_PATH);
        System.out.println(file.getAbsoluteFile());
        if (config==null){
            return (CONFIG_PATH==null || CONFIG_PATH.length()==0) ? ConfigFactory.load(): ConfigFactory.parseFile(new File(CONFIG_PATH));
        }
        return config;
    }
}

工具类

public class Util {

    //将当前日期格式化成字符串
    public static String getDateStr(){
        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmssSSS");
        String dateStr = sdf.format(new Date());
        return dateStr;
    }


    public static String getRandomHeader(){
        String[] userAgent ={
                "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)",
                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0",
                "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1",

                "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB7.0)",
                "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
                "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)",

                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36(KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36",

        };
        //随机生成一个在数组范围内的索引
        int index = new Random().nextInt(userAgent.length);
        return userAgent[index];
    }
}

主类

public class Wallpaper {
    //根据传入每页的url解析出资源的url
    public static void main(String[] args){
        Config config = AppConfig.getInstance();
        String url = config.getString("config.url");
        String path = config.getString("config.savePath");

        process(url, path);
    }

    private static void process(String url, String path) {

        //获取Document对象
        Document document = null;
        try {
            document = connect(url).get();
            System.out.println("页面->" + url);
            String title = document.getElementsByTag("title").first().text();
            System.out.println(title);

            Elements figures = document.getElementsByTag("figure");

            List<String> list = new ArrayList<String>();

            for (Element figure : figures) {
                String img = figure.select("img").attr("src");
                list.add(img);
            }

            for (String resourceUrl : list) {
                downloadResource(resourceUrl, path);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static void downloadResource(String resourceUrl, String savePath) throws IOException {

        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet(resourceUrl);

        //添加头部信息模拟浏览器访问
        //headergroup.updateHeader(new BasicHeader(name, value));
        //List<Header> headers   每个请求头的参数作为一个对象
        httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        httpGet.setHeader("Accept-Encoding", "gzip, deflate, sdch, br");
        httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8");
        httpGet.setHeader("User-Agent", Util.getRandomHeader());
        httpGet.setHeader("Referer", "http://www.baidu.com/");//设置来源

        //3.使用客户端执行请求,获取响应
        CloseableHttpResponse response = httpClient.execute(httpGet);

        int statusCode = response.getStatusLine().getStatusCode();
        if (statusCode == 200) {
            System.out.print("downloading...  ");
            System.out.print("resourceUrl -> " + resourceUrl+" ");

            //4.获取响应体
            HttpEntity entity = response.getEntity();
            //5.获取响应体的内容
            InputStream is = entity.getContent();
            //获取资源的后缀
            String suffixName = resourceUrl.substring(resourceUrl.lastIndexOf("."));
            //创建一个随时间毫秒值变化的的文件名
            String imgName = Util.getDateStr();
            imgName = imgName + suffixName;

            FileOutputStream fos = new FileOutputStream(savePath + "\\" + imgName);

            IOUtils.copy(is, fos);           //将输入流中的内容拷贝到输出流
            //关流
            fos.close();
            is.close();
            System.out.println("ok");
        }
    }
}

resource->log4j.xml

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="warn" monitorInterval="30" strict="true"
               schema="Log4J-V2.2.xsd">
    <Appenders>
        <Console name="Console" target="SYSTEM_OUT">
            <PatternLayout
                    pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n"/>
        </Console>

        <!--fileName的OracleLogToFBPlugin-main最好为启动类-->
        <RollingFile name="RollingFile" fileName="logs/wallPaper.log"
                     filePattern="logs/ZhiHu-wallpaper-%d{yyyy-MM-dd}-%i.log.gz">
            <PatternLayout
                    pattern="%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n"/>
            <Policies>
                <OnStartupTriggeringPolicy/>
                <TimeBasedTriggeringPolicy/>
                <SizeBasedTriggeringPolicy size="100 MB"/>
            </Policies>

            <DefaultRolloverStrategy max="20">
                <Delete basePath="logs/">
                    <IfFileName glob="logs/ZhiHu-wallpaper-*.log.gz"/>
                    <IfLastModified age="7D"/>
                </Delete>
            </DefaultRolloverStrategy>
        </RollingFile>
    </Appenders>

    <Loggers>
    <!--root的作用是是收集所有配置的logger(additivity=true)反馈上来的信息流并且根据root中配置的appender-ref进行输出,
    所以,如果我们logger配置了additivity=true,又配置了appender-ref,会导致日志重复。
        注意:这里的level值对那些没有配置日志级别,而且additivity=true的logger才会有作用。
    -->
        <Root level="debug">
             <AppenderRef ref="Console"/>
             <AppenderRef ref="RollingFile"/>
        </Root>
    </Loggers>
</Configuration>