使用网络抓取列出超出范围的索引

Question

我的代码出现列表索引超出范围错误。

import requests
from bs4 import BeautifulSoup
import re
import pyperclip
# import pandas as pd
import csv



# Get a name of the agency

def getAgency(pageURL):

    res = requests.get(pageURL)
    res.raise_for_status()

    soup = BeautifulSoup(res.text, 'html.parser')
    elems = soup.select('h3.company-name > a')

    names = []
    for i in range(len(elems)):
        names.append(str(elems[i].text.strip()))
    return names


def getWebsite(pageURL):

    res = requests.get(pageURL)
    res.raise_for_status()

    soup = BeautifulSoup(res.text, 'html.parser')
    elems = soup.select('li.website-link.website-link-a > a')

    sites = []
    for elem in elems:
        if elem.find('/your-project') != -1:
            elems.remove(elem)

        else:
            pass

    for i in range(len(elems)):
        sites.append(str(elems[i]["href"]))
    return sites

allNames = []
for pagenumber in range(0,1):
    names = getAgency('https://clutch.co/agencies/digital?page=' + str(pagenumber))
    allNames += names

allSites = []
for pagenumber in range(0,1):
    sites = getWebsite('https://clutch.co/agencies/digital?page=' + str(pagenumber))
    allSites += sites

final = []
with open('text.csv', 'w', newline='') as f:
    a = csv.writer(f, delimiter=',')
    for index in range(len(allNames)):
        final.append(",".join([allNames[index].replace(",", " "), allSites[index]]))
        a.writerows(final)

finalresult = "\n".join(final) 
pyperclip.copy(finalresult)

我知道是什么导致了这个错误，这个 if 语句。

for elem in elems:
        if elem.find('/your-project') != -1:
            elems.remove(elem)

        else:
            pass

当我从列表中删除一个项目时，索引号减1。因此，在这个for循环中，索引将是allNames的长度，我使用了与allSites相同的索引。因此，当索引将是 allNames 的最后一项时，allSites 将抛出错误，因为它超出范围。我该怎么做才能解决这个问题？

for index in range(len(allNames)):
            final.append(",".join([allNames[index].replace(",", " "), allSites[index]]))
            a.writerows(final)

Answer 1

您需要在一次迭代中解析代码并吐出成对的名称和站点，然后过滤以丢弃成对。

由于您的代码现在是，在您尝试按“/您的项目”过滤列表之前，列表的长度甚至都不一样，因为存在匹配 'li.website-link.wibise-link-a > a'

的广告链接

使用网络抓取列出超出范围的索引

List index out of range with web scraping

python

list

indexoutofrangeexception