无法验证网站，禁止访问

Question

问题

我正在尝试在此网站上验证自己的身份：https://www.peddemoeek.nl/

然而，到目前为止所有尝试都失败并返回了 ErrorCode Forbidden，WebClient 将其处理为 WebException.

环境

C#
提琴手
Visual Studio 2015
Chrome 浏览器
HTML 敏捷包

预期结果

网站响应登录成功

正常的浏览器登录结果

程序登录结果

代码

AdvancedWebClient.cs

public class AdvancedWebClient : WebClient
{
    public AdvancedWebClient()
    {
        this.CookieContainer = new CookieContainer();
    }

    public static NameValueCollection ParseFormFields(string aHTML)
    {
        if (aHTML == null)
        {
            throw new ArgumentNullException("Invalid argument");
        }

        HtmlDocument document = new HtmlDocument();
        document.OptionAutoCloseOnEnd = true;
        document.LoadHtml(aHTML);

        NameValueCollection result = new NameValueCollection();

        HtmlNodeCollection inputNodes = document.DocumentNode.SelectNodes("//input[@name]");
        foreach (HtmlNode node in inputNodes)
        {
            HtmlAttribute name = node.Attributes["name"];
            HtmlAttribute value = node.Attributes["value"];

            if (value == null)
            {
                result.Add(name.Value, string.Empty);
            }
            else
            {
                result.Add(name.Value, value.Value.Replace('`', '\"'));
            }
        }

        return result;
    }

    public static bool MatchXPath(string aHTML, string aXPath)
    {
        if (aHTML == null || aXPath == null)
        {
            throw new ArgumentNullException("Invalid argument(s)");
        }

        HtmlDocument document = new HtmlDocument();
        document.OptionAutoCloseOnEnd = true;
        document.LoadHtml(aHTML);

        return (document.DocumentNode.SelectSingleNode(aXPath) != null);
    }

    protected override WebRequest GetWebRequest(Uri aAddress)
    {
        HttpWebRequest request = base.GetWebRequest(aAddress) as HttpWebRequest;

        if (request != null)
        {
            request.CookieContainer = this.CookieContainer;
        }

        return request;
    }

    protected override WebResponse GetWebResponse(WebRequest aRequest)
    {
        WebResponse response = base.GetWebResponse(aRequest);

        this.ProcessResponse(response);

        return response;
    }

    protected override WebResponse GetWebResponse(WebRequest aRequest, IAsyncResult aResult)
    {
        WebResponse response = base.GetWebResponse(aRequest, aResult);

        this.ProcessResponse(response);

        return response;
    }

    private void ProcessResponse(WebResponse aResponse)
    {
        HttpWebResponse response = aResponse as HttpWebResponse;

        if (response != null)
        {
            foreach (Cookie cookie in response.Cookies)
            {
                this.CookieContainer.SetCookies(response.ResponseUri, cookie.Name + '=' + cookie.Value);
            }
        }
    }

    public CookieContainer CookieContainer
    {
        get;
        private set;
    }
}

SecuredWebsite.cs

public enum LoginResult
{
    LOGIN_RESULT_OK,
    LOGIN_RESULT_INVALID_CREDENTIALS,
    LOGIN_RESULT_ERROR
}

public abstract class SecuredWebsite : IDisposable
{
    public SecuredWebsite(string aBaseAddress)
    {
        if (aBaseAddress == null)
        {
            throw new ArgumentNullException("Invalid argument");
        }

        this.mClient = new AdvancedWebClient();
        this.disposedValue = false;

        this.mClient.Headers[HttpRequestHeader.AcceptCharset] = "UTF-8";
        this.mClient.BaseAddress = aBaseAddress;
    }

    public void Dispose()
    {
        Dispose(true);
    }

    public abstract LoginResult Login(string aUsername, string aPassword);

    protected virtual void Dispose(bool disposing)
    {
        if (this.disposedValue == false)
        {
            if (disposing)
            {
                this.mClient.Dispose();
            }

            this.disposedValue = true;
        }
    }

    protected AdvancedWebClient mClient;

    private bool disposedValue;
}

PeddemoeekWebsite.cs

public class PeddemoeekWebsite : SecuredWebsite
{
    public PeddemoeekWebsite() :
        base("https://www.peddemoeek.nl/")
    {
        base.mClient.Headers[HttpRequestHeader.CacheControl] = "max-age=0";
        base.mClient.Headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
        base.mClient.Headers[HttpRequestHeader.AcceptLanguage] = "nl-NL,nl;q=0.8,en-US;q=0.6,en;q=0.4";
        base.mClient.Headers[HttpRequestHeader.UserAgent] = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36";
        base.mClient.Headers[HttpRequestHeader.AcceptEncoding] = "";
        base.mClient.Headers["Upgrade-Insecure-Requests"] = "1";
        base.mClient.Headers[HttpRequestHeader.Referer] = "https://www.peddemoeek.nl/";
        base.mClient.Headers["Origin"] = "https://www.peddemoeek.nl";
    }

    public override LoginResult Login(string aUsername, string aPassword)
    {
        if (aUsername == null || aPassword == null)
        {
            throw new ArgumentNullException("Invalid argument(s)");
        }

        LoginResult result = LoginResult.LOGIN_RESULT_OK;

        try
        {
            byte[] responseData = base.mClient.DownloadData("/");
            if (responseData == null)
            {
                result = LoginResult.LOGIN_RESULT_ERROR;
            }
            else
            {
                string response = Encoding.UTF8.GetString(responseData);

                NameValueCollection formFields = AdvancedWebClient.ParseFormFields(response);

                formFields.Set("username", aUsername);
                formFields.Set("passwd", aPassword);
                formFields.Set("remember", "no");
                formFields.Set("Submit", "");

                responseData = base.mClient.UploadValues("/component/comprofiler/login.html", formFields);
                if (responseData == null)
                {
                    result = LoginResult.LOGIN_RESULT_ERROR;
                }
                else
                {
                    response = Encoding.UTF8.GetString(responseData);
                    // TODO: handle response
                }
            }
        }
        catch (WebException)
        {
            result = LoginResult.LOGIN_RESULT_ERROR;
        }

        return result;
    }
}

Answer 1

显然通过 WebClient 的 Headers 属性设置用户代理并没有设置它。我不得不把它放在 WebRequest 级别：

protected override WebRequest GetWebRequest(Uri aAddress)
    {
        HttpWebRequest request = base.GetWebRequest(aAddress) as HttpWebRequest;

        if (request != null)
        {
            request.CookieContainer = this.CookieContainer;
            request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36";
            request.Headers[HttpRequestHeader.AcceptCharset] = "UTF-8";
        }

        return request;
    }

无法验证网站，禁止访问

Unable to authenticate to website, Forbidden

.net

c#

authentication

html-agility-pack

问题

环境

预期结果

正常的浏览器登录结果

程序登录结果

代码