JAVA网络爬虫
HttpClient

导航

 

项目结构

在这里插入图片描述
在这里插入图片描述

配置文件

  1. pom.xml(父模块)

    <?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <modelVersion>4.0.0</modelVersion>
    
        <groupId>com.xiaoge.demo</groupId>
        <artifactId>cloud-demo</artifactId>
        <version>1.0-SNAPSHOT</version>
        <modules>
            <module>user-service</module>
            <module>consumer-demo</module>
            <module>eureka-server</module>
        </modules>
        <packaging>pom</packaging>
    
        <parent>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-parent</artifactId>
            <version>2.0.4.RELEASE</version>
            <relativePath/> <!-- lookup parent from repository -->
        </parent>
    
        <properties>
            <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
            <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
            <java.version>1.8</java.version>
            <spring-cloud.version>Finchley.SR1</spring-cloud.version>
            <mapper.starter.version>2.0.3</mapper.starter.version>
            <mysql.version>5.1.32</mysql.version>
        </properties>
    
        <dependencyManagement> <!-- 所有子工程, 必须引下面的依赖才会有 -->
            <dependencies>
                <!-- spring cloud 引用了它, 以后cloud依赖版本就不用写了 -->
                <dependency>
                    <groupId>org.springframework.cloud</groupId>
                    <artifactId>spring-cloud-dependencies</artifactId>
                    <version>${spring-cloud.version}</version>
                    <type>pom</type>
                    <scope>import</scope>
                </dependency>
    
                <!-- 通用Mapper启动器 -->
                <dependency>
                    <groupId>tk.mybatis</groupId>
                    <artifactId>mapper-spring-boot-starter</artifactId>
                    <version>${mapper.starter.version}</version>
                </dependency>
    
                <!-- mysql驱动 -->
                <dependency>
                    <groupId>mysql</groupId>
                    <artifactId>mysql-connector-java</artifactId>
                    <version>${mysql.version}</version>
                </dependency>
    
            </dependencies>
        </dependencyManagement>
    
        <dependencies> <!-- 因为不会再dependencyManagement标签里, 所以所有的子工程都可以用 -->
            <dependency>
                <groupId>org.projectlombok</groupId>
                <artifactId>lombok</artifactId>
            </dependency>
        </dependencies>
    
        <build>
            <plugins>
                <plugin>
                    <groupId>org.springframework.boot</groupId>
                    <artifactId>spring-boot-maven-plugin</artifactId>
                </plugin>
            </plugins>
        </build>
    
    </project>
    
  2. pom.xml(子模块: user-service)

    <?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <parent>
            <artifactId>cloud-demo</artifactId>
            <groupId>com.xiaoge.demo</groupId>
            <version>1.0-SNAPSHOT</version>
        </parent>
        <modelVersion>4.0.0</modelVersion>
    
        <artifactId>user-service</artifactId>
    
        <dependencies>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
    
            <dependency>
                <groupId>mysql</groupId>
                <artifactId>mysql-connector-java</artifactId>
            </dependency>
    
            <dependency>
                <groupId>tk.mybatis</groupId>
                <artifactId>mapper-spring-boot-starter</artifactId>
            </dependency>
    
            <!-- 引入eureka(注册中心)-客户端 -->
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
            </dependency>
        </dependencies>
    
    </project>
    
  3. pom.xml(子模块: eureka-server)

    <?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <parent>
            <artifactId>cloud-demo</artifactId>
            <groupId>com.xiaoge.demo</groupId>
            <version>1.0-SNAPSHOT</version>
        </parent>
        <modelVersion>4.0.0</modelVersion>
    
        <artifactId>eureka-server</artifactId>
    
        <dependencies>
            <!-- springcloud 注册中心 eureka -->
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-starter-netflix-eureka-server</artifactId>
            </dependency>
        </dependencies>
    
    </project>
    
  4. pom.xml(子模块: consumer-demo)

    <?xml version="1.0" encoding="UTF-8"?>
    <project xmlns="http://maven.apache.org/POM/4.0.0"
             xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
             xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
        <parent>
            <artifactId>cloud-demo</artifactId>
            <groupId>com.xiaoge.demo</groupId>
            <version>1.0-SNAPSHOT</version>
        </parent>
        <modelVersion>4.0.0</modelVersion>
    
        <artifactId>consumer-demo</artifactId>
    
        <dependencies>
            <dependency>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-starter-web</artifactId>
            </dependency>
    
            <!-- 引入eureka客户端依赖 -->
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
            </dependency>
        </dependencies>
    
    </project>
    
  5. application.yml(子模块: user-service)

    server:
      port: 9090
    spring:
      # 在eureka中注册的服务名称
      application:
        name: user-service
      # 数据库连接信息配置
      datasource:
        url: jdbc:mysql://localhost:3306/yun6
        username: root
        password: root
        hikari:
          maximum-pool-size: 20
          minimum-idle: 10
    # 别名包
    mybatis:
      type-aliases-package: com.xiaoge.user.pojo
    
    # 注册方
    # eureka配置 因为它要去注册中心注册, 所以要配置
    eureka:
      client:
        service-url:
          defaultZone: http://127.0.0.1:8761/eureka
    
  6. application.yml(子模块: eureka-server)

    # 这是eureka的默认端口号
    server:
      port: 8761
    
    # 配置eureka服务名称
    spring:
      application:
        name: eureka-server
    
    # 注册方
    # 告诉eureka要注册的地址在哪里 (这样eureka就不会报一个com.sun.jersey.api.client.ClientHandlerException异常)
    eureka:
      client:
        service-url:
          defaultZone: http://127.0.0.1:8761/eureka
      instance:
        prefer-ip-address: true # 我希望使用ip地址
        ip-address: 127.0.0.1 # 设置ip地址
    
    
  7. application.yml(子模块: consumer-demo)

    server:
      port: 8088
    spring:
      # 在eureka中注册的服务名称
      application:
        name: consumer-service
    
    # 注册方
    # eureka配置 因为它要去注册中心注册, 所以要配置
    eureka:
      client:
        service-url:
          defaultZone: http://127.0.0.1:8761/eureka
    

实现类

  1. User(子模块: user-service)

    package com.xiaoge.user.pojo;
    
    import lombok.Data;
    import tk.mybatis.mapper.annotation.KeySql;
    
    import javax.persistence.Id;
    import javax.persistence.Table;
    import java.io.Serializable;
    import java.util.Date;
    
    /**
     * @program: cloud-demo
     * @description:
     * @author: Mr.Xiao
     * @create: 2020-05-04 15:46
     **/
    @Table(name = "tb_user")
    @Data
    public class User implements Serializable {
    
        @Id
        @KeySql(useGeneratedKeys = true) // id自动增长
        private Long id;
    
        // 用户名
        private String username;
    
        // 密码
        private String password;
    
        // 姓名
        private String name;
    
        // 年龄
        private Integer age;
    
        // 性别,1男性,2女性
        private Integer sex;
    
        // 出生日期
        private Date birthday;
    
        // 创建时间
        private Date created;
    
        // 更新时间
        private Date updated;
    
        // 备注
        private String note;
    
    }
    
    
    
  2. User(子模块: consumer-demo)(注意: 因为他要调用user-service系统资源所以要写一个实体类来映射数据)

    package com.xiaoge.Consumer.pojo;
    
    import lombok.Data;
    
    import java.io.Serializable;
    import java.util.Date;
    
    /**
     * @program: cloud-demo
     * @description:
     * @author: Mr.Xiao
     * @create: 2020-05-04 16:06
     **/
    @Data
    public class User implements Serializable {
        // 主键
        private Long id;
    
        // 用户名
        private String username;
    
        // 密码
        private String password;
    
        // 姓名
        private String name;
    
        // 年龄
        private Integer age;
    
        // 性别,1男性,2女性
        private Integer sex;
    
        // 出生日期
        private Date birthday;
    
        // 创建时间
        private Date created;
    
        // 更新时间
        private Date updated;
    
        // 备注
        private String note;
    
    }
    
    

持久层

  1. UserMapper(子模块: user-service)

    package com.xiaoge.user.mapper;
    
    
    import com.xiaoge.user.pojo.User;
    import tk.mybatis.mapper.common.Mapper;
    
    /**
     * @program: cloud-demo
     * @description:
     * @author: Mr.Xiao
     * @create: 2020-05-04 15:49
     **/
    
    public interface UserMapper extends Mapper<User> {
    }
    
    

业务层

  1. UserService接口(子模块: user-service)

    package com.xiaoge.user.service;
    
    import com.xiaoge.user.pojo.User;
    
    /**
     * @program: cloud-demo
     * @description:
     * @author: Mr.Xiao
     * @create: 2020-05-04 15:50
     **/
    public interface UserService {
    
        public User queryById(Long id);
    
    }
    
    
  2. UserServiceImpl实现类(子模块: user-service)

    package com.xiaoge.user.service.impl;
    
    import com.xiaoge.user.mapper.UserMapper;
    import com.xiaoge.user.pojo.User;
    import com.xiaoge.user.service.UserService;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.stereotype.Service;
    
    /**
     * @program: cloud-demo
     * @description:
     * @author: Mr.Xiao
     * @create: 2020-05-04 15:51
     **/
    @Service
    public class UserServiceImpl implements UserService {
    
        @Autowired
        private UserMapper userMapper;
    
        @Override
        public User queryById(Long id) {
            return userMapper.selectByPrimaryKey(id);
        }
    }
    

表现层

  1. UserController(子模块: user-service)

    package com.xiaoge.user.controller;
    
    import com.xiaoge.user.pojo.User;
    import com.xiaoge.user.service.UserService;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.web.bind.annotation.GetMapping;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.RestController;
    
    /**
     * @program: cloud-demo
     * @description:
     * @author: Mr.Xiao
     * @create: 2020-05-04 15:51
     **/
    @RestController
    @RequestMapping("/user")
    public class UserController {
    
        @Autowired
        private UserService userService;
    
        @GetMapping("/{id}")
        public User queryById(@PathVariable("id") Long id) {
            User user = userService.queryById(id);
            return user;
        }
    
    }
    
    
  2. ConsumerController(子模块: consumer-demo)

    package com.xiaoge.Consumer.controller;
    
    import com.xiaoge.Consumer.pojo.User;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.cloud.client.ServiceInstance;
    import org.springframework.cloud.client.discovery.DiscoveryClient;
    import org.springframework.web.bind.annotation.GetMapping;
    import org.springframework.web.bind.annotation.PathVariable;
    import org.springframework.web.bind.annotation.RequestMapping;
    import org.springframework.web.bind.annotation.RestController;
    import org.springframework.web.client.RestTemplate;
    
    import java.util.List;
    
    /**
     * @program: cloud-demo
     * @description: 动态拉取服务列表
     * @author: Mr.Xiao
     * @create: 2020-05-04 16:08
     **/
    @RestController
    @RequestMapping("/consumer")
    public class ConsumerController {
    
        @Autowired
        private RestTemplate restTemplate;
    
        @Autowired
        private DiscoveryClient discoveryClient; // 专门负责服务发现
    
        @GetMapping("/{id}")
        public User queryById(@PathVariable("id") Long id){
            // 拉取服务  根据服务id获取实例
            List<ServiceInstance> instances = discoveryClient.getInstances("user-service");
    
            // 获取第一个实例, 从实例当中取出ip和端口
            ServiceInstance instance = instances.get(0); // 这个位置正常情况下, 要写负载均衡算法, 我这里是随便取得
    
            // 获取ip
            String ip = instance.getHost();
    
            // 获取端口
            int port = instance.getPort();
    
            // 访问的地址
            String url = "http://" + ip + ":" + port;
    
            User user = restTemplate.getForObject(url + "/user/" + id, User.class);
            return user;
        }
    
    }
    

引导类

  1. UserApplication(子模块: user-service)

    package com.xiaoge;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
    import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
    import tk.mybatis.spring.annotation.MapperScan;
    
    /**
     * @program: cloud-demo
     * @description: 服务的提供方
     * @author: Mr.Xiao
     * @create: 2020-05-04 15:43
     **/
    @EnableDiscoveryClient // 它也是客户端, 它捷能兼容eureka 又能consul、zookeeper等等, 更通用, 更广泛
    @SpringBootApplication
    @MapperScan("com.xiaoge.user.mapper")
    public class UserApplication {
    
        public static void main(String[] args) {
            SpringApplication.run(UserApplication.class);
        }
    
    }
    
    
    
  2. EurekaServer(子模块: eureka-server)

    package com.xiaoge;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.cloud.netflix.eureka.server.EnableEurekaServer;
    
    /**
     * @program: cloud-demo
     * @description: EurekaServer端
     * @author: Mr.Xiao
     * @create: 2020-05-04 16:34
     **/
    @EnableEurekaServer // 启动Eureka服务
    @SpringBootApplication
    public class EurekaServer {
    
        public static void main(String[] args) {
            SpringApplication.run(EurekaServer.class);
        }
    
    }
    
    
  3. ConsumerApplication(子模块: consumer-demo)

    package com.xiaoge;
    
    import org.springframework.boot.SpringApplication;
    import org.springframework.boot.autoconfigure.SpringBootApplication;
    import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
    import org.springframework.context.annotation.Bean;
    import org.springframework.web.client.RestTemplate;
    
    /**
     * @program: cloud-demo
     * @description: 服务调用方
     * @author: Mr.Xiao
     * @create: 2020-05-04 16:04
     **/
    @EnableDiscoveryClient // 它也是客户端, 它捷能兼容eureka 又能consul、zookeeper等等, 更通用, 更广泛
    @SpringBootApplication
    public class ConsumerApplication {
    
        @Bean // 把它加载进spring容器
        public RestTemplate restTemplate() {
            return new RestTemplate();
        }
    
        public static void main(String[] args) {
            SpringApplication.run(ConsumerApplication.class);
        }
    
    }
    
    
posted on 2020-05-04 20:43  gmlgxx  阅读(39)  评论(0)    收藏  举报