了解使用 python 3.8 和请求进行网络抓取的承载授权

Question

所以我想抓取以下网站：

https://hyland.csod.com/ux/ats/careersite/4/home?c=hyland

我运行使用 Python 请求库的原因是 header 需要我传递带有某种令牌的授权 header .如果我手动转到该页面，复制并粘贴它，然后运行我的程序，我可以让它工作，但我想知道如何绕过这个问题（毕竟，运行如果我仍然需要手动访问实际站点并检索授权令牌，则使用抓取工具）。

我是授权/持有人 headers 的新手，我希望有人能够阐明浏览器如何生成令牌来检索此信息/我如何模拟此信息。这是我的代码：

import requests
import json
import datetime

today = datetime.datetime.today()

url = "https://hyland.csod.com/services/x/career-site/v1/search"

# actual sitehttps://hyland.csod.com/ux/ats/careersite/4/home?c=hyland

headers = {
    'authority': 'hyland.csod.com',
    'origin': 'https://hyland.csod.com',
    'authorization': 'Bearer eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCIsImNsaWQiOiI0bDhnbnFhbGk3NjgifQ.eyJzdWIiOi0xMDMsImF1ZCI6IjRxNTFzeG5oY25yazRhNXB1eXZ1eGh6eCIsImNvcnAiOiJoeWxhbmQiLCJjdWlkIjoxLCJ0emlkIjoxNCwibmJkIjoiMjAxOTEyMzEyMTE0MTU5MzQiLCJleHAiOiIyMDE5MTIzMTIyMTUxNTkzNCIsImlhdCI6IjIwMTkxMjMxMjExNDE1OTM0In0.PlNdWXtb1uNoMuGIhI093ZbheRN_DwENTlkNoVr0j7Zah6JHd5cukudVFnZEiQmgBZ_nlDU4C-9JO_2We380Vg',
    'content-type': 'application/json',
    'accept': 'application/json; q=1.0, text/*; q=0.8, */*; q=0.1',
    'x-requested-with': 'XMLHttpRequest',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
    'csod-accept-language': 'en-US',
    'referer': 'https://hyland.csod.com/ux/ats/careersite/4/home?c=hyland',
    'accept-encoding': 'gzip, deflate, br',
    'cookie': 'CYBERU_lastculture=en-US; ASP.NET_SessionId=4q51sxnhcnrk4a5puyvuxhzx; cscx=hyland^|-103^|1^|14^|KumB4VhzYXML22MnMxjtTB9SKgHiWW0tFg0HbHnOek4=; c-s=expires=1577909201~access=/clientimg/hyland/*^!/content/hyland/*~md5=78cd5252d2efff6eb77d2e6bf0ce3127',
}



data = ['{"careerSiteId":4,"pageNumber":1,"pageSize":25,"cultureId":1,"searchText":"","cultureName":"en-US","states":["oh"],"countryCodes":[],"cities":[],"placeID":"","radius":null,"postingsWithinDays":null,"customFieldCheckboxKeys":[],"customFieldDropdowns":[],"customFieldRadios":[]}',
        '{"careerSiteId":4,"pageNumber":2,"pageSize":25,"cultureId":1,"searchText":"","cultureName":"en-US","states":["oh"],"countryCodes":[],"cities":[],"placeID":"","radius":null,"postingsWithinDays":null,"customFieldCheckboxKeys":[],"customFieldDropdowns":[],"customFieldRadios":[]}']

def hyland(url, data):
    # for openings in data:

    dirty = requests.post(url, headers=headers, data=data).text

    if 'Unauthorized' in dirty:
        print(dirty)
        print("There was an error connecting. Check Info")

    # print(dirty)
    clean = json.loads(dirty)
    cleaner = json.dumps(clean, indent=4)
    print("Openings at Hyland Software in Westlake as of {}".format(today.strftime('%m-%d-%Y')))
    for i in range(0,60):
        try:
            print(clean["data"]["requisitions"][i]["displayJobTitle"])
            print("")
            print("")
        except:
            print("{} Openings at Hyland".format(i))
            break

for datum in data:    
    hyland(url, data=datum)

所以基本上我的代码正在做的是向上面的 url 发送一个 post 请求以及 header 和必要的数据来检索我想要的。这个爬虫的工作时间很短，但如果我离开并在几个小时后回来，它就会因为授权而不再工作（至少我是这样得出结论的）。

任何有关这一切如何工作的帮助/说明将不胜感激。

Answer 1

您的代码有几个问题：

如您所述，您必须获得不记名令牌
您必须使用 requests.session() 发送请求（因为此网页似乎会注意您发送的 cookie）
可选：您的 headers 有很多不必要的 headers 可以删除

总而言之，下面是工作代码：

import requests
import json
import datetime

today = datetime.datetime.today()
session = requests.session()

url = "https://hyland.csod.com:443/ux/ats/careersite/4/home?c=hyland"
headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate", "DNT": "1", "Connection": "close", "Upgrade-Insecure-Requests": "1"}

raw = session.get(url, headers=headers).text
token = raw[raw.index("token")+8:]
token = token[:token.index("\"")]
bearer_token = f"Bearer {token}"

url = "https://hyland.csod.com/services/x/career-site/v1/search"

# actual sitehttps://hyland.csod.com/ux/ats/careersite/4/home?c=hyland

headers =  {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0", "Authorization": bearer_token}

data = ['{"careerSiteId":4,"pageNumber":1,"pageSize":25,"cultureId":1,"searchText":"","cultureName":"en-US","states":["oh"],"countryCodes":[],"cities":[],"placeID":"","radius":null,"postingsWithinDays":null,"customFieldCheckboxKeys":[],"customFieldDropdowns":[],"customFieldRadios":[]}',
        '{"careerSiteId":4,"pageNumber":2,"pageSize":25,"cultureId":1,"searchText":"","cultureName":"en-US","states":["oh"],"countryCodes":[],"cities":[],"placeID":"","radius":null,"postingsWithinDays":null,"customFieldCheckboxKeys":[],"customFieldDropdowns":[],"customFieldRadios":[]}']

def hyland(url, data, session= session):
    # for openings in data:

    dirty = session.post(url, headers=headers, data=data).text

    if 'Unauthorized' in dirty:
        print(dirty)
        print("There was an error connecting. Check Info")

    # print(dirty)
    clean = json.loads(dirty)
    cleaner = json.dumps(clean, indent=4)
    print("Openings at Hyland Software in Westlake as of {}".format(today.strftime('%m-%d-%Y')))
    for i in range(0,60):
        try:
            print(clean["data"]["requisitions"][i]["displayJobTitle"])
            print("")
            print("")
        except:
            print("{} Openings at Hyland".format(i))
            break

for datum in data:    
    hyland(url, data=datum, session = session)

希望这对您有所帮助

了解使用 python 3.8 和请求进行网络抓取的承载授权

Understanding Bearer Authorization for web scraping using python 3.8 and requests

authorization

web-scraping

python-3.x

python-requests

bearer-token