java

Question

问题是使用多线程时cookie被覆盖了。

我有一个包含 getHtmlResult 方法的 WebBot 接口来发送请求并获得 html 页面结果：

import java.util.Date;

public interface WebBot {
    public String getHtmlResult(Date pickedDate) throws Exception;
}

两个工具是WebBotA和WebBotB： WebBotA:

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.CookieHandler;
import java.net.CookieManager;
import java.net.CookiePolicy;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.zip.GZIPInputStream;

import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Service;

@Service("webBotA")
@Scope("prototype")
public class WebBotA implements WebBot{
    String urlHeader = "http://booknow.jetstar.com";
    String urlTailBeforeRedirect = "/Search.aspx?culture=vi-VN";
    String charset = "UTF-8";
    String requestMethod = "POST";
    int readTimeOut = 10000;

    @Override
    public String getHtmlResult(Date pickedDate) throws Exception {
        System.out.println("WebBot A started");

        String htmlResult = "";

        DateFormat df = new SimpleDateFormat("dd");
        String pickedDateDay = df.format(pickedDate);

        df = new SimpleDateFormat("yyyy-MM");
        String pickedDateMonth = df.format(pickedDate);


        Map<String, String> requestParams = initRequestParams();
        String postParameters = "ControlGroupSearchView%24AvailabilitySearchInputSearchView%24DropDownListCurrency=VND&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24DropDownListFareTypes=I&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24"
                + "DropDownListMarketDay1=" + pickedDateDay
                + "&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24DropDownListMarketDay2=1&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24DropDownListMarketDay3=&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24"
                + "DropDownListMarketMonth1=" + pickedDateMonth
                + "&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24DropDownListMarketMonth2=1968-1&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24DropDownListMarketMonth3=&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24"
                + "DropDownListPassengerType_ADT=" + 1
                + "&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24"
                + "DropDownListPassengerType_CHD=" + 0
                + "&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24"
                + "DropDownListPassengerType_INFANT=" + 0
                + "&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24RadioButtonMarketStructure=OneWay&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24"
                + "TextBoxMarketDestination1=" + "HAN"
                + "&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24TextBoxMarketDestination2=&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24TextBoxMarketDestination3=&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24"
                + "TextBoxMarketOrigin1=" + "SGN"
                + "&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24TextBoxMarketOrigin2=&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24TextBoxMarketOrigin3=&ControlGroupSearchView%24ButtonSubmit=&__VIEWSTATE=%2FwEPDwUBMGQYAQUeX19Db250cm9sc1JlcXVpcmVQb3N0QmFja0tleV9fFgEFJ01lbWJlckxvZ2luU2VhcmNoVmlldyRtZW1iZXJfUmVtZW1iZXJtZSDCMtVG%2F1lYc7dy4fVekQjBMvD5&culture=vi-VN&date_picker=&go-booking=&pageToken=sLkmnwXwAsY%3D&ControlGroupSearchView%24AvailabilitySearchInputSearchView%24fromCS=yes";

            try {
                // Cookie manager registry
                CookieHandler.setDefault(new CookieManager(null, CookiePolicy.ACCEPT_ALL));

                URL obj = new URL(urlHeader + urlTailBeforeRedirect);
                HttpURLConnection conn = (HttpURLConnection) obj.openConnection();

                // Set redirect to false
                conn.setInstanceFollowRedirects(false);
                conn.setReadTimeout(readTimeOut);
                conn.setRequestMethod(requestMethod);
                for (Map.Entry<String, String> requestParamEntry : requestParams.entrySet()) {
                    conn.setRequestProperty(requestParamEntry.getKey(), requestParamEntry.getValue());
                }

                // Send post request
                conn.setDoOutput(true);
                DataOutputStream wr = new DataOutputStream(conn.getOutputStream());
                wr.writeBytes(postParameters);
                wr.flush();
                wr.close();

                // get redirect url from "location" header field
                String urlTailAfterRedirect = conn.getHeaderField("Location");

                // get the cookie if need, for login
                String cookies = conn.getHeaderField("Set-Cookie");

                // open the new connnection again
                conn = (HttpURLConnection) new URL(urlHeader + urlTailAfterRedirect).openConnection();
                conn.setRequestProperty("Cookie", cookies);

                conn.setReadTimeout(readTimeOut);
                conn.setRequestMethod(requestMethod);
                for (Map.Entry<String, String> requestParamEntry : requestParams.entrySet()) {
                    conn.setRequestProperty(requestParamEntry.getKey(), requestParamEntry.getValue());
                }

                conn.setDoOutput(true);
                wr = new DataOutputStream(conn.getOutputStream());
                wr.writeBytes(postParameters);
                wr.flush();
                wr.close();

                String inputLine;

                InputStream connectionInputStream = conn.getInputStream();
                String contentEncoding = conn.getContentEncoding();
                if(contentEncoding != null){
                    if (contentEncoding.toLowerCase().contains("gzip")) {
                        connectionInputStream = new GZIPInputStream(connectionInputStream);
                    }   
                }
                else{
                    throw new IOException("Failed to get Content Encoding. May by cause Cookie Manager");
                }

                BufferedReader in = new BufferedReader(new InputStreamReader(connectionInputStream, charset));

                StringBuffer html = new StringBuffer();

                while ((inputLine = in.readLine()) != null) {
                    html.append(inputLine);
                }
                in.close();

                htmlResult =html.toString();

            }
            catch (IOException e) {
                System.out.println(e.toString());
            }
            finally{
            }
            return htmlResult;
        }

    private Map<String, String> initRequestParams(){
        Map<String, String> requestParams = new HashMap<String, String>();
        requestParams.put("User-Agent", "Mozilla/5.0");
        requestParams.put("Accept-Language", "en-US,en;q=0.5");
        requestParams.put("Content-Type", "application/x-www-form-urlencoded");
        requestParams.put("Accept-Charset", charset);
        requestParams.put("Host", "booknow.jetstar.com");
        requestParams.put("Referer", "http://www.jetstar.com/vn/vi/home");
        requestParams.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        requestParams.put("Accept-Encoding", "gzip, deflate");
        requestParams.put("Connection", "keep-alive");
        return requestParams;
    }
}

WebBotB:

@Service("webBotB")
@Scope("prototype")
public class WebBotB implements WebBot{

    String urlNoSessionId = "https://ameliaweb5.intelisys.ca/VietJet/ameliapost.aspx?lang=en";
    String urlWithSessionId = "https://ameliaweb5.intelisys.ca/VIETJET/TravelOptions.aspx?lang=en&st=pb&sesid=";
    String charset = "UTF-8";
    String requestPostMethod = "POST";
    String requestGetMethod = "GET";
    int readTimeOut = 10000;

    @Override
    public String getHtmlResult(Date pickedDate) throws Exception {

        System.out.println("WebBot B started");
        String htmlResult = "";

        DateFormat df = new SimpleDateFormat("dd");
        String departDayStr = df.format(pickedDate);

        df = new SimpleDateFormat("yyyy-MM");
        String departMonthStr = df.format(pickedDate).replaceAll("-", "%2F");

        Date currentDate = new Date();
        df = new SimpleDateFormat("dd");
        String currentDayStr = df.format(currentDate);
        df = new SimpleDateFormat("yyyy-MM");
        String currentMonthStr = df.format(currentDate).replaceAll("-", "%2F");

        Map<String, String> requestParams = initRequestParams();
        String postParameterRequest1 = "chkRoundTrip=&"
                + "lstOrigAP=" + "SGN"
                + "&lstDestAP=" + "HAN"
                + "&dlstDepDate_Day=" + departDayStr
                + "&dlstDepDate_Month=" + departMonthStr
                + "&dlstRetDate_Day=" + departDayStr
                + "&dlstRetDate_Month=" + departMonthStr
                + "&lstCurrency=VND&lstResCurrency=VND&lstDepDateRange=0&lstRetDateRange=0&"
                + "txtNumAdults=" + 1
                + "&txtNumChildren=" + 0
                + "&txtNumInfants=" + 0
                + "&lstLvlService=1&blnFares=False&txtPromoCode=";

        String postParameterRequest2 = "__VIEWSTATE=%2FwEPDwULLTE1MzQ1MjI3MzAPZBYCZg9kFg4CCA8QZGQWAGQCCQ8QZGQWAGQCCw8QZGQWAGQCDQ8QZGQWAGQCEQ8QZGQWAGQCEg8QZGQWAGQCEw8QZGQWAGRkDuhQN17CT5ZIydlFFSt%2BWc8NsCA%3D&__VIEWSTATEGENERATOR=BA3C3B49&SesID=&DebugID=62&lstOrigAP=-1&lstDestAP=-1&"
                + "dlstDepDate_Day=" + currentDayStr
                + "&dlstDepDate_Month=" + currentMonthStr
                + "&lstDepDateRange=0&dlstRetDate_Day=" + currentDayStr
                + "&dlstRetDate_Month=" + departMonthStr
                + "&lstRetDateRange=0"
                + "&txtNumAdults=0"
                + "&txtNumChildren=0"
                + "&txtNumInfants=0"
                + "&lstLvlService=1&lstResCurrency=VND&lstCurrency=VND&txtPromoCode=";

        try{
            CookieHandler.setDefault(null);

            // Begin step 1
            URL objectURLForRequest1_2 = new URL(urlNoSessionId);
            HttpURLConnection connection1 = (HttpURLConnection) objectURLForRequest1_2.openConnection();

            // Set redirect to false
            connection1.setInstanceFollowRedirects(false);
            connection1.setReadTimeout(readTimeOut);
            connection1.setRequestMethod(requestPostMethod);
            for (Map.Entry<String, String> requestParamEntry : requestParams.entrySet()) {
                connection1.setRequestProperty(requestParamEntry.getKey(), requestParamEntry.getValue());
            }
            connection1.setFixedLengthStreamingMode(postParameterRequest1.length());

            // Send post request
            connection1.setDoOutput(true);
            DataOutputStream dataOutputStream = new DataOutputStream(connection1.getOutputStream());
            dataOutputStream.writeBytes(postParameterRequest1);
            dataOutputStream.flush();
            dataOutputStream.close();

            String cookieInRequest1String = connection1.getHeaderField("Set-Cookie");

            // Get session id for request 3
            String sessionIdForRequest3 = "";
            List<HttpCookie> cookies = HttpCookie.parse(cookieInRequest1String);
            for (HttpCookie httpCookie : cookies) {
                if (httpCookie.getName().equals("ASP.NET_SessionId"))
                {
                    sessionIdForRequest3 = httpCookie.getValue();
                    break;
                }
            }

            // Begin step 2
            HttpURLConnection connection2 = (HttpURLConnection) objectURLForRequest1_2.openConnection();
            // Set redirect to false
            connection2.setInstanceFollowRedirects(false);
            connection2.setReadTimeout(readTimeOut);
            connection2.setRequestMethod(requestPostMethod);
            for (Map.Entry<String, String> requestParamEntry : requestParams.entrySet()) {
                connection2.setRequestProperty(requestParamEntry.getKey(), requestParamEntry.getValue());
            }
            connection2.setRequestProperty("Cookie", cookieInRequest1String);
            connection2.setFixedLengthStreamingMode(postParameterRequest2.length());

            // Send post request
            connection2.setDoOutput(true);
            dataOutputStream = new DataOutputStream(connection2.getOutputStream());
            dataOutputStream.writeBytes(postParameterRequest2);
            dataOutputStream.flush();
            dataOutputStream.close();

            // Ommit this connection2.getInputStream will lead to Session Expired !
            @SuppressWarnings("unused")
            InputStream connectionInputStream2 = connection2.getInputStream();

            // Begin step 3
            urlWithSessionId = urlWithSessionId + sessionIdForRequest3;
            URL objectURLForRequest3 = new URL(urlWithSessionId);
            HttpURLConnection connection3 = (HttpURLConnection) objectURLForRequest3.openConnection();
            connection3.setRequestMethod(requestGetMethod);
            connection3.setRequestProperty("Cookie", cookieInRequest1String);
            for (Map.Entry<String, String> requestParamEntry : requestParams.entrySet()) {
                connection3.setRequestProperty(requestParamEntry.getKey(), requestParamEntry.getValue());
            }

            String inputLine;

            InputStream connectionInputStream = connection3.getInputStream();
            String contentEncoding = connection3.getContentEncoding();
            if (connection3.getContentEncoding() != null){
                if (contentEncoding.toLowerCase().contains("gzip")) {
                    connectionInputStream = new GZIPInputStream(connectionInputStream);
                }
                if (contentEncoding.toLowerCase().contains("deflate")){
                    connectionInputStream = new DeflaterInputStream(connectionInputStream);
                }
            }

            BufferedReader in = new BufferedReader(new InputStreamReader(connectionInputStream, charset));

            StringBuffer html = new StringBuffer();

            while ((inputLine = in.readLine()) != null) {
                html.append(inputLine);
            }
            in.close();

            htmlResult = html.toString();

        }
        catch (IOException e) {
            System.out.println(e.toString());
        }
        finally{
        }
        return htmlResult;
    }

    private Map<String, String> initRequestParams(){
        Map<String, String> requestParams = new HashMap<String, String>();
        requestParams.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2329.0 Safari/537.36");
        requestParams.put("Accept-Language", "en-US,en;q=0.5");
        requestParams.put("Content-Type", "application/x-www-form-urlencoded");
        requestParams.put("Accept-Charset", charset);
        requestParams.put("Host", "ameliaweb5.intelisys.ca");
        requestParams.put("Referer", "https://ameliaweb5.intelisys.ca/VietJet/ameliapost.aspx?lang=vi");
        requestParams.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        requestParams.put("Accept-Encoding", "gzip, deflate");
        requestParams.put("Connection", "keep-alive");
        return requestParams;
    }
}

我使用 WebBotThread.java 将 Runnable 实现到运行多个线程：

import java.util.Date;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.AutowireCapableBeanFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import core.WebBot;
import core.WebBotA;
import core.WebBotB;

@Component("webBotThread")
@Scope("prototype")
public class WebBotThread implements Runnable{
    @Autowired 
    AutowireCapableBeanFactory factory;

    private Date pickedDate;
    private int webBotType;

    public WebBotThread(){}

    @Override
    public void run() {
        WebBot webBot = null;
        if (webBotType == 1)
        {
            webBot = (WebBotA)factory.getBean("webBotA");
        }
        else{
            webBot = (WebBotB)factory.getBean("webBotB");
        }

        try {
            webBot.getHtmlResult(pickedDate);
            if(webBotType == 1)
                System.out.println("WebBotA finished successfully");
            else
                System.out.println("WebBotB finished successfully");
        } catch (Exception e) {
            if(webBotType == 1)
                System.out.println("WebBotA failed: " + e.toString());
            else
                System.out.println("WebBotB failed: " + e.toString());
        }
    }

    public Date getPickedDate() {
        return pickedDate;
    }

    public void setPickedDate(Date pickedDate) {
        this.pickedDate = pickedDate;
    }

    public int getWebBotType() {
        return webBotType;
    }

    public void setWebBotType(int webBotType) {
        this.webBotType = webBotType;
    }

}

我的Main.javaclass:

import java.util.Calendar;
import java.util.Date;

import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

public class Main {

    public static void main(String[] args) {
        try {
            @SuppressWarnings("resource")
            ApplicationContext context = new ClassPathXmlApplicationContext( "classpath:spring/applicationContext.xml");

            Calendar cal = Calendar.getInstance();
            cal.set(Calendar.YEAR, 2015);
            cal.set(Calendar.MONTH, Calendar.APRIL);
            cal.set(Calendar.DAY_OF_MONTH, 30);
            Date pickedDate = cal.getTime();

            ThreadPoolTaskExecutor threadPoolTaskExecutor = (ThreadPoolTaskExecutor) context.getBean("ticketFinderBoExecutor");

            WebBotThread wbThreadA = context.getBean(WebBotThread.class);
            wbThreadA.setWebBotType(1);
            wbThreadA.setPickedDate(pickedDate);

            WebBotThread wbThreadB = context.getBean(WebBotThread.class);
            wbThreadB.setWebBotType(2);
            wbThreadB.setPickedDate(pickedDate);

            threadPoolTaskExecutor.execute(wbThreadA);
            threadPoolTaskExecutor.execute(wbThreadB);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

}

我的applicationContext-thread.xml设置：

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:aop="http://www.springframework.org/schema/aop"
    xmlns:tx="http://www.springframework.org/schema/tx" xmlns:context="http://www.springframework.org/schema/context"
    xsi:schemaLocation="
    http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
    http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.1.xsd
    http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd
    http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd"
    default-autowire="byName">

    <bean id="ticketFinderBoExecutor"
        class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
        <property name="corePoolSize" value="2" />
        <property name="maxPoolSize" value="2" />
        <property name="WaitForTasksToCompleteOnShutdown" value="true" />
    </bean>
</beans>

我的applicationContext.xml

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:aop="http://www.springframework.org/schema/aop"
    xmlns:tx="http://www.springframework.org/schema/tx" xmlns:context="http://www.springframework.org/schema/context"
    xsi:schemaLocation="
    http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
    http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.1.xsd
    http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd
    http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd"
    default-autowire="byName">
    <context:spring-configured/>

    <import resource="applicationContext-thread.xml" />

    <!-- Declare for annotation -->
    <context:annotation-config />

    <!-- Declare for transaction manager -->
    <tx:annotation-driven />

    <!-- Auto scan Beans to Spring container -->
    <context:component-scan
        base-package="core
                    , helper
                    , thread" />
</beans>

而我的问题是，当我在 applicationContext-thread.xml 中仅设置 1 个线程池时，两个 WebBotA 和 WebBotB 运行 OK。但是，当我同时将 2 个线程池设置为运行 WebBotA 和 WebBotB 时，两个 WebBot 工具之一将出现异常。我发现这个 topic of Wurstbro 可能和我有同样的问题，但是，当我将他的解决方案应用到我的上下文时，我也失败了。我已经研究了很多，但仍然没有找到解决我的问题的方法。请帮助我，非常感谢。

P/s: 原谅我英语不好

Answer 1

您在 URL 实例上调用 openConnection。这将返回一个 HttpUrlConnection 实例。

默认Java 将仅通过标准 UrlStreamHandlerFactory 创建一个 UrlStreamHandler 一次。这意味着，如果您的应用程序使用多个 UrlConnection，它始终使用相同的 UrlStreamHandler。这就是您的 cookie 被共享的原因。

如何修复？

或者您创建自己的 UrlStreamHandlerFactory 并强制使用 ThreadLocals 来获得不同的 cookie。对我来说似乎并不容易。

或者你切换到另一个库来处理URLs。我鼓励使用 Apache Http Client 库中的 HttpClient。如果您实例化 2 个 HttpClient，它们将不会共享 cookie。

据我所知，您使用的是 Spring 框架，使用 Apache Http Client 库绝对不会有问题。

如果你在执行它时遇到困难，请大声喊叫！

java - cookie 在多线程中被覆盖

java - cookie overried in multi-thread

multithreading

cookies

cookiemanager