Python通过代理使用多线程爬取安居客二手房数据（二）

本文主要是介绍Python通过代理使用多线程爬取安居客二手房数据（二），希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

在上一篇文章python使用代理爬取安居客二手房数据（一）上增加爬取内容
爬取内容为：：‘待售房屋’, ‘室’, ‘厅’, ‘卫’, ‘面积’, ‘面积单位’, ‘朝向’, ‘楼层’, ‘建筑年份’,
‘小区名称’, ‘区’, ‘镇’, ‘道路’, ‘标签’, ‘总价’, ‘总价单位’, ‘均价’, ‘均价单位’ 并使用多线程提高爬取速度

爬取网址管理器

"""
@author rubyw
爬虫的url管理器
"""class CrawlerUrlManager():def __init__(self):self.new_urls = set()self.old_urls = set()# 新增一个待爬取Urldef add_new_url(self, url):if url is None or len(url) == 0:returnif url in self.new_urls or url in self.old_urls:returnself.new_urls.add(url)return True# 批量新增urldef add_new_urls(self, urls):if urls is None or len(urls) == 0:returnfor url in urls:self.add_new_url(url)# 获取一个要爬取的urldef get_url(self):if self.has_new_url():url = self.new_urls.pop()self.old_urls.add(url)return urlelse:return None# 批量获取待爬取的urldef get_new_urls(self, num):returnUrls = set()if num is None or type(num) != int or num <= 0:return returnUrlselse:i = 0while self.has_new_url() and i < num:url = self.new_urls.pop()self.old_urls.add(url)returnUrls.add(url)i = i + 1return returnUrls# 判断是否有待爬取的urldef has_new_url(self):return len(self.new_urls) > 0# 获取待爬取url的数量def get_new_url_size(self):return len(self.new_urls)# 获取已爬取url的数量def get_old_url_size(self):return len(self.old_urls)if __name__ == "__main__":url_manager = CrawlerUrlManager()# 添加两个url，批量添加故意添加一个重复的url，看去重是否okurl_manager.add_new_url("url1")url_manager.add_new_urls(["url1", "url2"])print(url_manager.new_urls, url_manager.old_urls)# 获取一个url，然后打印两个集合print("#" * 30)new_url = url_manager.get_url()print(url_manager.new_urls, url_manager.old_urls)# 再获取一个url，然后打印两个集合print("#" * 30)new_url = url_manager.get_url()print(url_manager.new_urls, url_manager.old_urls)# 看看两个集合中还有没有Urlprint("#" * 30)print(url_manager.has_new_url())

爬取二手房数据

"""
爬取安居客网站苏州的二手房数据
启动方法：进入/anjuke目录下，执行python secondhand_house_crawler.py
"""import requests
from bs4 import BeautifulSoup
import threading
import time
import csv
from crawlUrlManager import CrawlerUrlManagerdef get_proxies():proxy_list = []proxy_url = ''  # 替换成自己的try:datas = requests.get(proxy_url).json()print(datas['code'])# 如果代理ip获取成功if datas['code'] == 0:proxy_list = datas['data']['proxy_list']# data_array = datas['data']# for i in range(len(data_array)):#     proxy_ip = data_array[i]['ip']#     proxy_port = str(data_array[i]['port'])#     proxy = proxy_ip + ":" + proxy_port#     proxy_list.append({'http': 'http://' + proxy, 'https': 'http://' + proxy})else:code = datas['code']print(f'获取代理失败，状态码={code}')return proxy_listexcept Exception as e:# print('调用天启API获取代理IP异常:' + e)print('调用快代理API获取代理IP异常:' + e)return proxy_listdef craw_anjuke_wuhan(craw_url, proxy):if craw_url is None:print(threading.current_thread().getName() + ' craw_url is None')return# 用户名密码认证(私密代理/独享代理)username = ""  # 替换成自己的password = ""  # 替换成自己的proxies = {"http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {'user': username, 'pwd': password,'proxy': proxy},"https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {'user': username, 'pwd': password,'proxy': proxy}}print(proxies)print(threading.current_thread().getName() + f' is crawing {craw_url}...使用代理{proxy}')# 构造url的request headers，伪装成正常用户headers = {'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7','accept-encoding': 'gzip, deflate, br','accept-language': 'zh-CN,zh;q=0.9','cache-control': 'no-cache','cookie': 'aQQ_ajkguid=70C0288A-42CB-4C56-B8EF-8E90F8077A8C; sessid=13C76F04-9178-4EE8-B8B0-F00FE21F4F50; ajk-appVersion=; ctid=22; fzq_h=d23302afd92c82b304657a734e3950aa_1697613588983_b645e9292cff4c148c0e3fb2ff31662e_3746354997; id58=CrIej2Uvhxc/D8k8IRI2Ag==; twe=2; fzq_js_anjuke_ershoufang_pc=8e86fa86290dbac07d5de51dd3b9db13_1697615100824_23; obtain_by=1; xxzl_cid=817f908b661647889fa49debaab80d9c; xxzl_deviceid=lrdQ4FRXrfXyN2Qj/gRhBw2SQpTZ81igKeOBCkzlfzjPwEG8whpE1uKNvVqIOvXQ','host': 'wuhan.anjuke.com','pragma': 'no-cache','referer': 'https://wuhan.anjuke.com/sale/jianghana/p1/','sec-ch-ua': '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"','sec-ch-ua-mobile': '?0','sec-ch-ua-platform': "Windows",'sec-fetch-dest': 'document','sec-fetch-mode': 'navigate','sec-fetch-site': 'same-origin','sec-fetch-user': '?1','upgrade-insecure-requests': '1','user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36'}with open('data/wuhanSecondHouse.csv', 'a', newline='', encoding="gbk") as f:# 有代理用代理，没代理直接爬if proxy is None:r = requests.get(craw_url, headers=headers, timeout=3)else:r = requests.get(craw_url, headers=headers, proxies=proxies, timeout=3)# 如果正常返回结果，开始解析if r.status_code == 200:content = r.text# print(content)soup = BeautifulSoup(content, 'html.parser')content_div_nodes = soup.find_all('div', class_='property-content')for content_div_node in content_div_nodes:# 获取房产标题内容content_title_name = content_div_node.find('h3', class_='property-content-title-name')title_name = content_title_name.get_text()# 获取房子户型content_layout = content_div_node.find('p',class_='property-content-info-text property-content-info-attribute')layout_datas = content_layout.find_all('span')datas_shi = layout_datas[0].get_text() + layout_datas[1].get_text()datas_ting = layout_datas[2].get_text() + layout_datas[3].get_text()datas_wei = layout_datas[4].get_text() + layout_datas[5].get_text()# 获取房子的面积、朝向、楼层和建筑年份square_num = ''square_unit = ''orientations = ''floor_level = ''build_year = ''content_extra_info_datas = content_div_node.find_all(lambda content_div_node: content_div_node.name == 'p' and content_div_node.get('class') == ['property-content-info-text'])for i in range(len(content_extra_info_datas)):if i == 0:square = content_extra_info_datas[0].get_text().strip()square_num = square[0:len(square) - 1]square_unit = square[len(square) - 1:]if i == 1:orientations = content_extra_info_datas[1].get_text().strip()if i == 2:floor_level = content_extra_info_datas[2].get_text().strip()if i == 3:build_year = content_extra_info_datas[3].get_text().strip()# 获取房子的小区名称、位置信息（区-镇-道路）content_info_comm = content_div_node.find('div',class_='property-content-info property-content-info-comm')# 获取小区名称housing_estate = content_info_comm.find('p',class_='property-content-info-comm-name').get_text().strip()# 获取小区地址信息content_info_address = content_info_comm.find('p',class_='property-content-info-comm-address').find_all('span')district = content_info_address[0].get_text().strip()town = content_info_address[1].get_text().strip()road = content_info_address[2].get_text().strip()# 获取房子的更多tag信息，比如朝向、是否满五唯一、房子新旧、是否近地铁等content_info_tag = content_div_node.find_all('span', class_='property-content-info-tag')tagstr = ''for i in range(len(content_info_tag)):tagstr = tagstr + content_info_tag[i].get_text().strip() + ','# 获取房子价格信息price_info_datas = content_div_node.find('div', class_='property-price')total_price = price_info_datas.find('span', class_='property-price-total-num').get_text().strip()total_price_unit = price_info_datas.find('span', class_='property-price-total-text').get_text().strip()avarage_price = price_info_datas.find('p', class_='property-price-average').get_text().strip()avarage_price_num = avarage_price[0:len(avarage_price) - 3]avarage_price_unit = avarage_price[len(avarage_price) - 3:]# 输出到文件writer = csv.writer(f)writer.writerow([title_name, datas_shi, datas_ting, datas_wei, square_num, square_unit, orientations, floor_level,build_year, housing_estate, district, town, road, tagstr, total_price, total_price_unit,avarage_price_num, avarage_price_unit])# f.write("%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s;%s\n" % (# title_name, datas_shi, datas_ting, datas_wei, square_num, square_unit, orientations, floor_level,# build_year, housing_estate, district, town, road, tagstr, total_price, total_price_unit,# avarage_price_num, avarage_price_unit))print(f'{threading.current_thread().getName()} crawl over!;Crawler Url is:{craw_url}')else:print(f'{threading.current_thread().getName()} crawl fail!status code={r.status_code};Crawler Url is:{craw_url}')if __name__ == '__main__':# 先将标题写入结果数据文件with open('data/wuhanSecondHouse.csv', 'w', newline='', encoding='gbk') as f:writer = csv.writer(f)writer.writerow(['待售房屋', '室', '厅', '卫', '面积', '面积单位', '朝向', '楼层', '建筑年份', '小区名称', '区', '镇', '道路', '标签', '总价', '总价单位', '均价','均价单位'])# 假设爬取crawler_pages页，生成待爬取的url，放入url池管理起来crawlerUrlManager = CrawlerUrlManager()# 要爬取的页数，默认为100，可调整crawler_pages = 200for i in range(crawler_pages):region = "jianghana"url = 'https://wuhan.anjuke.com/sale/{region}/p{page}/'craw_url = url.format(region=region, page=i)crawlerUrlManager.add_new_url(craw_url)# 尝试获取代理ip，避免同一个ip频繁访问被网站的反爬机制给封禁proxy_list = get_proxies()proxy_num = len(proxy_list)if proxy_num >= 2:  # 如果获取到代理ip，则用代理ip，建议至少获取5个及以上的代理ip，爬取的时候每个线程一个ip进行爬取print(f'获取到{proxy_num}个代理ip，开始使用代理IP爬取页面数据...')while crawlerUrlManager.has_new_url():crawler_threads = []for i in range(len(proxy_list)):proxy = proxy_list[i]print(crawlerUrlManager.get_url())crawler_thread = threading.Thread(craw_anjuke_wuhan(crawlerUrlManager.get_url(), proxy))crawler_threads.append(crawler_thread)# 启动线程开始爬取for crawler_thread in crawler_threads:crawler_thread.start()for crawler_thread in crawler_threads:crawler_thread.join()# 谨慎起见，一批线程爬取结束后，间隔一段时间，再启动下一批爬取，这里默认设置为3秒，可调整time.sleep(3)else:  # 如果没获取到代理ip，则直接爬取，控制一下每个线程爬取的间隔时间，不要太频繁try:print('没有获取到代理IP，开始使用自身IP爬取页面数据...')while crawlerUrlManager.has_new_url():crawler_thread = threading.Thread(target=craw_anjuke_wuhan, args=(crawlerUrlManager.get_url(), None))crawler_thread.start()crawler_thread.join()time.sleep(10)  # 为避免同一个ip频繁爬取被反爬封禁，一线程爬取完后，等待10秒再爬取下一个页面except Exception as e:print('Crawler Excepiton:' + e)finally:print(f'已爬取的url数量：{crawlerUrlManager.get_old_url_size()}')print(f'未爬取的url数量：{+crawlerUrlManager.get_new_url_size()}')if crawlerUrlManager.get_new_url_size() > 0:print('未爬取的url如下：')for new_url in crawlerUrlManager.get_url():print(f'{new_url}')