SpringBoot集成okhttp3爬虫

    科技2024-07-02  67

    一.引入依赖

    pom依赖如下:

    <dependency> <groupId>com.squareup.okhttp3</groupId> <artifactId>okhttp</artifactId> <version>3.10.0</version> </dependency>

    二.在resources目录下配置application.properties

    # 配置okhttp3 ok.http.connect-timeout=30 ok.http.read-timeout=30 ok.http.write-timeout=30 # 连接池中整体的空闲连接的最大数量 ok.http.max-idle-connections=200 # 连接空闲时间最多为 300 秒 ok.http.keep-alive-duration=300

    三.创建conf包,在该包下创建OkHttpConfiguration配置类

    import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import okhttp3.ConnectionPool; import okhttp3.OkHttpClient; @Configuration public class OkHttpConfiguration { @Value("${ok.http.connect-timeout}") private Integer connectTimeout; @Value("${ok.http.read-timeout}") private Integer readTimeout; @Value("${ok.http.write-timeout}") private Integer writeTimeout; @Value("${ok.http.max-idle-connections}") private Integer maxIdleConnections; @Value("${ok.http.keep-alive-duration}") private Long keepAliveDuration; @Bean public OkHttpClient okHttpClient() { return new OkHttpClient.Builder() .sslSocketFactory(sslSocketFactory(), x509TrustManager()) // 是否开启缓存 .retryOnConnectionFailure(false) .connectionPool(pool()) .connectTimeout(connectTimeout, TimeUnit.SECONDS) .readTimeout(readTimeout, TimeUnit.SECONDS) .writeTimeout(writeTimeout,TimeUnit.SECONDS) .hostnameVerifier((hostname, session) -> true) // 设置代理 // .proxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 8888))) // 拦截器 // .addInterceptor() .build(); } @Bean public X509TrustManager x509TrustManager() { return new X509TrustManager() { @Override public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { } @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { } @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } }; } @Bean public SSLSocketFactory sslSocketFactory() { try { // 信任任何链接 SSLContext sslContext = SSLContext.getInstance("TLS"); sslContext.init(null, new TrustManager[]{x509TrustManager()}, new SecureRandom()); return sslContext.getSocketFactory(); } catch (NoSuchAlgorithmException | KeyManagementException e) { e.printStackTrace(); } return null; } @Bean public ConnectionPool pool() { return new ConnectionPool(maxIdleConnections, keepAliveDuration, TimeUnit.SECONDS); } }

    四.使用工具类快速完成解析

    package com.okhttpcilentofspringbootl.springbootinit.utils; import java.io.IOException; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; public class OkHttpUtils { public static String OkGetArt(String url){ String html = null; OkHttpClient client = new OkHttpClient(); Request requestObj = new Request.Builder().url(url).build(); try (Response responseObj = client.newCall(requestObj).execute()){ html = responseObj.body().string(); } catch (IOException e) { e.printStackTrace(); } return html; } }
    Processed: 0.010, SQL: 9