Скребок в BeautifulSoup 4 с Python - новичок

Question

Спасибо за последний вопрос (here). Тем не менее, я в настоящее время застреваю в том, чтобы получить окончательный файл данных. Мне удалось извлечь все данные из исходной таблицы и отобразить ее, как я хочу, однако теперь я хочу добавить домашнюю команду и команду гостей в df, но, похоже, не в состоянии понять это. Вот то, что у меня есть сейчас, и here is the site Я хочу соскоблить.

from urllib.request import urlopen # import the library 
from bs4 import BeautifulSoup # Import BS 
from bs4 import SoupStrainer # Import Soup Strainer 
import pandas as pd # import pandas as a package 

basescrape = 'http://www.footywire.com/afl/footy/ft_match_statistics?mid=' 
matchid = '6172' 

scrapeweb1 = basescrape+matchid 

page = urlopen(scrapeweb1) # access the website 
only_tables = SoupStrainer('table', attrs={"width" : "583"}) # parse only table elements when parsing 
soup = BeautifulSoup(page, 'html.parser', parse_only=only_tables) # parse the html 

only_teams = SoupStrainer('table', attrs={"width" : "376"}) # parse only team qtr score elements when parsing 
soup2 = BeautifulSoup(page, 'html.parser', parse_only=only_teams) # parse the html 


# only valid rows with player data in 

table = soup.find_all("tr", attrs={"onmouseover" : "this.bgColor='#cbcdd0';"}) 


# create variables to keep the data in 

hometeam = [] 
awayteam = [] 

player = [] 
kicks = [] 
handballs = [] 
disposals = [] 
marks = [] 
goals = [] 
behinds = [] 
tackles = [] 
hitouts = [] 
inside50s = [] 
freesfor = [] 
freesagainst = [] 
fantasy = [] 
supercoach = [] 

# Find all the <tr> tag pairs, skip the first one, then for each. 
for row in soup.find_all("tr", attrs={"onmouseover" : "this.bgColor='#cbcdd0';"}): 
    # Create a variable of all the <td> tag pairs in each <tr> tag pair, 
    col = row.find_all('td') 

    # Create a variable of the string inside 1st <td> tag pair, 
    column_1 = col[0].string.strip() 
    # and append it to player variable 
    player.append(column_1) 

    # Create a variable of the string inside 2nd <td> tag pair, 
    column_2 = col[1].string.strip() 
    # and append it to kicks variable 
    kicks.append(column_2) 

    # Create a variable of the string inside 3rd <td> tag pair, 
    column_3 = col[2].string.strip() 
    # and append it to handballs variable 
    handballs.append(column_3) 

    # Create a variable of the string inside 4th <td> tag pair, 
    column_4 = col[3].string.strip() 
    # and append it to disposals variable 
    disposals.append(column_4) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_5 = col[4].string.strip() 
    # and append it to marks variable 
    marks.append(column_5) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_6 = col[5].string.strip() 
    # and append it to goals variable 
    goals.append(column_6) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_7 = col[6].string.strip() 
    # and append it to behinds variable 
    behinds.append(column_7) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_8 = col[7].string.strip() 
    # and append it to tackles variable 
    tackles.append(column_8) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_9 = col[8].string.strip() 
    # and append it to hitouts variable 
    hitouts.append(column_9) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_10 = col[9].string.strip() 
    # and append it to inside50s variable 
    inside50s.append(column_10) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_11 = col[10].string.strip() 
    # and append it to freesfo variable 
    freesfor.append(column_11) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_12 = col[11].string.strip() 
    # and append it to freesagainst variable 
    freesagainst.append(column_12) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_13 = col[12].string.strip() 
    # and append it to fantasy variable 
    fantasy.append(column_13) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_14 = col[13].string.strip() 
    # and append it to supercoach variable 
    supercoach.append(column_14) 

# Find all the <tr> tag pairs, then for each. 
for row in soup2.find_all("tr", class_= "leftbold"): 
    # Create a variable of all the <td> tag pairs in each <tr> tag pair, 
    col2 = row.find_all('td') 

    # Create a variable of the string inside 1st <td> tag pair, 
    hometeam = col2[0].string.strip() 
    # and append it to player variable 
    # hometeam.append(column2_1) 

    # Create a variable of the string inside 2nd <td> tag pair, 
    awayteam = col2[1].string.strip() 
    # and append it to kicks variable 
    # awayteam.append(column2_2) 


# Create a variable of the value of the columns 
columns = {'match_id': matchid, 'home_team': hometeam, 'away_team': awayteam, 'player': player, 'kicks': kicks, 'handballs': handballs, 'disposals': disposals, 'marks': marks, 'goals': goals, 'behinds': behinds, 'tackles': tackles, 'hitouts': hitouts, 'inside_50s': inside50s, 'frees_for': freesfor, 'frees_against': freesagainst, 'fantasy': fantasy, 'supercoach': supercoach} 

# Create a dataframe from the columns variable - n 
df = pd.DataFrame(columns, columns = ['match_id', 'home_team', 'away_team', 'player', 'kicks', 'handballs', 'disposals', 'marks', 'goals', 'behinds', 'tackles', 'hitouts', 'inside_50s', 'frees_for', 'frees_against', 'fantasy', 'supercoach']) 

print(df) 

# print(soup.prettify()) 

# print(table) 

Очевидно, что dataframe не будет работать, поскольку массивы не имеют одинаковой длины. Как я могу очистить домашнюю и гостевую команду и сохранить ее в переменной, чтобы она работала так же, как работает маттид?

Изменен ли способ, чтобы переменная «hometeam» отображалась в первых 22 строках и «awayteam» в 23-44-м рядах? таким образом, игрок приписывается одной команде?

Я чувствую, что я делаю этот раздел здесь неправильно:

# Find all the <tr> tag pairs, then for each. 
for row in soup2.find_all("tr", class_= "leftbold"): 
    # Create a variable of all the <td> tag pairs in each <tr> tag pair, 
    col2 = row.find_all('td') 

    # Create a variable of the string inside 1st <td> tag pair, 
    hometeam = col2[0].string.strip() 
    # and append it to player variable 
    # hometeam.append(column2_1) 

    # Create a variable of the string inside 2nd <td> tag pair, 
    awayteam = col2[1].string.strip() 
    # and append it to kicks variable 
    # awayteam.append(column2_2) 

Большое спасибо за вашу помощь.

(также вопрос с бонусом, я не мог заставить scrapeweb1 работать с использованием оператора «.join», так как я читал, используя строки «+» на строках, это не лучшая практика. Что я пытался использовать, но отказавший ниже)

scrapeweb1 = "".join(basescrape, matchid) 

Edit: так что я проверил источник и, кажется, есть некоторые неправильно HTML в этой таблице ...

<table border="0" cellspacing="0" cellpadding="0" width="376" id="matchscoretable"> 
<tr> 
<th class="leftbold" height="23" width="100">Team</td> 

он использует «/ TD», а не "/th ", который при анализе через красивый суп заставляет таблицу теги закрываться ...

[<table border="0" cellpadding="0" cellspacing="0" id="matchscoretable" width="376"> 
<tr> 
<th class="leftbold" height="23" width="100">Team</th></tr></table>] 

Я, возможно, придется искать другой способ получения дома и на выезде команд имена

Answer 1

Вот один из способов мог бы это сделать:

from urllib.request import urlopen # import the library 
from bs4 import BeautifulSoup # Import BS 
from bs4 import SoupStrainer # Import Soup Strainer 
import pandas as pd # import pandas as a package 

basescrape = 'http://www.footywire.com/afl/footy/ft_match_statistics?mid=' 
matchid = '6172' 
url = ''.join([basescrape,matchid]) 

# changed the table width to 585 to get first row with team name 
only_tables = SoupStrainer('table', attrs={"width" : "585"}) # parse only table elements when parsing 
soup = BeautifulSoup(urlopen(url), 'html.parser', parse_only=only_tables) # parse the html 
# use the table titles as anchor points 
teams = soup.find_all('td', attrs={'class':'innertbtitle', 'align':'left'}) 
# create an empty list for the players 
player_list = [] 
# iterate through anchor points 
for team in teams: 
    # extract team name from the table title 
    team_name = team.text.strip().split(' ', maxsplit=1)[0] 
    # get the rows from the next table relative to anchor point 
    trs = team.find_next('table', attrs={'width':583}).find_all('tr') 
    # create list of labels using first row in table 
    labels = [td.text for td in trs.pop(0).find_all('td')] 
    # iterate through the remaining rows 
    for row in trs: 
     # build dictionary using label as key and text of each cell as value 
     player_dict = {label:value.text for label,value in 
         zip(labels, row.find_all('td'))} 
     # add team name to dictionary 
     player_dict['team'] = team_name 
     # append dictionary to the list 
     player_list.append(player_dict) 

# create the dataframe 
df = pd.DataFrame(player_list) 
print(df) 

Answer 2

мне удалось решить эту проблему, вот завершенный код теперь ...

from urllib.request import urlopen # import the library 
from bs4 import BeautifulSoup # Import BS 
from bs4 import SoupStrainer # Import Soup Strainer 
import pandas as pd # import pandas as a package 

basescrape = 'http://www.footywire.com/afl/footy/ft_match_statistics?mid=' 
matchid = '6172' 

scrapeweb1 = basescrape+matchid 

page = urlopen(scrapeweb1) # access the website 
page2 = urlopen(scrapeweb1) # access the website 

only_tables = SoupStrainer('table', attrs={"width" : "583"}) # parse only table elements when parsing 

soup = BeautifulSoup(page, 'html.parser', parse_only=only_tables) # parse the html 
soup2 = BeautifulSoup(page2, 'html.parser') # parse the html 


# only valid rows with player data in 

table = soup.find_all("tr", attrs={"onmouseover" : "this.bgColor='#cbcdd0';"}) 


# create variables to keep the data in 

Table1 = soup2.find_all('table', attrs={'width':"375"})[1] 

hometeam = Table1.find_all('td', attrs={'width':"124"})[0].string.strip() 
awayteam = Table1.find_all('td', attrs={'width':"124"})[1].string.strip() 


player = [] 
kicks = [] 
handballs = [] 
disposals = [] 
marks = [] 
goals = [] 
behinds = [] 
tackles = [] 
hitouts = [] 
inside50s = [] 
freesfor = [] 
freesagainst = [] 
fantasy = [] 
supercoach = [] 

# Find all the <tr> tag pairs, skip the first one, then for each. 
for row in soup.find_all("tr", attrs={"onmouseover" : "this.bgColor='#cbcdd0';"}): 
    # Create a variable of all the <td> tag pairs in each <tr> tag pair, 
    col = row.find_all('td') 

    # Create a variable of the string inside 1st <td> tag pair, 
    column_1 = col[0].string.strip() 
    # and append it to player variable 
    player.append(column_1) 

    # Create a variable of the string inside 2nd <td> tag pair, 
    column_2 = col[1].string.strip() 
    # and append it to kicks variable 
    kicks.append(column_2) 

    # Create a variable of the string inside 3rd <td> tag pair, 
    column_3 = col[2].string.strip() 
    # and append it to handballs variable 
    handballs.append(column_3) 

    # Create a variable of the string inside 4th <td> tag pair, 
    column_4 = col[3].string.strip() 
    # and append it to disposals variable 
    disposals.append(column_4) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_5 = col[4].string.strip() 
    # and append it to marks variable 
    marks.append(column_5) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_6 = col[5].string.strip() 
    # and append it to goals variable 
    goals.append(column_6) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_7 = col[6].string.strip() 
    # and append it to behinds variable 
    behinds.append(column_7) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_8 = col[7].string.strip() 
    # and append it to tackles variable 
    tackles.append(column_8) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_9 = col[8].string.strip() 
    # and append it to hitouts variable 
    hitouts.append(column_9) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_10 = col[9].string.strip() 
    # and append it to inside50s variable 
    inside50s.append(column_10) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_11 = col[10].string.strip() 
    # and append it to freesfo variable 
    freesfor.append(column_11) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_12 = col[11].string.strip() 
    # and append it to freesagainst variable 
    freesagainst.append(column_12) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_13 = col[12].string.strip() 
    # and append it to fantasy variable 
    fantasy.append(column_13) 

    # Create a variable of the string inside 5th <td> tag pair, 
    column_14 = col[13].string.strip() 
    # and append it to supercoach variable 
    supercoach.append(column_14) 




# Create a variable of the value of the columns 
columns = {'match_id': matchid, 'home_team': hometeam, 'away_team': awayteam, 'player': player, 'kicks': kicks, 'handballs': handballs, 'disposals': disposals, 'marks': marks, 'goals': goals, 'behinds': behinds, 'tackles': tackles, 'hitouts': hitouts, 'inside_50s': inside50s, 'frees_for': freesfor, 'frees_against': freesagainst, 'fantasy': fantasy, 'supercoach': supercoach} 

# Create a dataframe from the columns variable - n 
df = pd.DataFrame(columns, columns = ['match_id', 'home_team', 'away_team', 'player', 'kicks', 'handballs', 'disposals', 'marks', 'goals', 'behinds', 'tackles', 'hitouts', 'inside_50s', 'frees_for', 'frees_against', 'fantasy', 'supercoach']) 

print(df)