Проблема с BeautifulSoup и запросы разбора

Question

Я получаю сообщение об ошибке при попытке использовать BeautifulSoup и модуль запросов.

Мой код выглядит следующим образом:

import requests 
from bs4 import BeautifulSoup 

def get_html(url): 
    url = ('https://m.vk.com/bageto?act=members&offset=0') 
    r = requests.get(url) 
    return r.text 

def get_total_pages(get_html): 
    soup = BeautifulSoup(get_html, 'lxml') 
    pages = soup.find('div', class_='pagination').find_all('a', class_='pg_link')[-1].get('href') 
    total_pages = pages.split('=')[2] 
    return int(total_pages) 

def main(): 
    base_url = 'https://m.vk.com/bageto?act=members&offset='`enter code here` 
    total_pages = get_total_pages(get_html) 
    for i in range(50, total_pages, 50): 
     print (i) 

Это дает ошибку:

C:\Users\PANDEMIC\Desktop\Python-Test>vkp.py Traceback (most recent call last): 
File "C:\Users\PANDEMIC\Desktop\Python-Test\vkp.py", 
line 23, in <module> 
    main() 
File "C:\Users\PANDEMIC\Desktop\Python-Test\vkp.py", line 20, in main 
    total_pages = get_total_pages(get_html) 
File "C:\Users\PANDEMIC\Desktop\Python-Test\vkp.py", line 13, in get_total_pages 
    soup = BeautifulSoup(get_html, 'lxml') 
File "C:\Users\PANDEMIC\AppData\Local\Programs\Python\Python36-32\lib\site-packages\bs4\__init__.py", line 192, in __init__ 
    elif len(markup) <= 256 and (
TypeError: object of type 'function' has no len() 

Answer 1

def main(): 
    try: 
     urll = [] 
     base_url = 'https://m.vk.com/bageto?act=members&offset=' 
     total_pages = int(get_total_pages(get_html(url))) 
     for i in range(0, total_pages, 50): 
      url_gen = str(base_url + str(i)) 
      urll.append(url_gen) 
      #get_page_data(url_gen) 
     pool = ThreadPool(8) 
     results = pool.map(get_page_data, urll) 

    except KeyboardInterrupt: 
     print('you are stopped script yourself') 

if __name__ == '__main__': 

    main() 

Answer 2

Вы забыли () и аргумент при выполнении get_html() в

total_pages = get_total_pages(get_html(base_url)) 

---

КСТАТИ: вам не нужно url в get_html, потому что он будет ovewrites своего аргумента в следующих вызовах

def get_html(url): 
    #url = ('https://m.vk.com/bageto?act=members&offset=0') 
    r = requests.get(url) 
    return r.text 

или вы можете использовать значение по умолчанию

def get_html(url='https://m.vk.com/bageto?act=members&offset=0') 
    r = requests.get(url) 
    return r.text 

Полной версии с base_url+"0" в качестве аргумента в get_html(base_url+"0")

import requests 
from bs4 import BeautifulSoup 

def get_html(url): 
    #url = ('https://m.vk.com/bageto?act=members&offset=0') 
    r = requests.get(url) 
    return r.text 

def get_total_pages(html): 
    soup   = BeautifulSoup(html, 'lxml') 
    pages   = soup.find('div', class_='pagination').find_all('a', class_='pg_link')[-1].get('href') 
    total_pages  = pages.split('=')[2] 
    return int(total_pages) 

def main(): 
    base_url = 'https://m.vk.com/bageto?act=members&offset=' 
    total_pages = get_total_pages(get_html(base_url+"0")) 

    print(total_pages) 

    for i in range(50, total_pages, 50): 
     print(i) 
     #print(base_url + str(i)) 

main() 

Answer 3

import requests 
from bs4 import BeautifulSoup 


def get_html(url): 
    url = ('https://m.vk.com/bageto?act=members&offset=0') 
    r = requests.get(url) 
    return r.text 

def get_total_pages(html): 

    soup   = BeautifulSoup(html, 'lxml') 
    pages   = soup.find('div', class_='pagination').find_all('a', class_='pg_link')[-1].get('href') 
    total_pages  = pages.split('=')[2] 
    return int(total_pages) 

def main(): 
    base_url = 'https://m.vk.com/bageto?act=members&offset=0' 
    html = get_html(base_url) 
    total_pages = get_total_pages(html) 
    print(total_pages) 

Вы должны передать строку html в BeautifulSoup, а не в функцию.