万方表面数据爬取

import requests,pandas as pd,time,re,json,random
from lxml import etree
from fake_useragent import UserAgent

D=['上海','南京','无锡','常州','苏州','南通','扬州','镇江','盐城','泰州']
L=['杭州','宁波','温州','湖州','嘉兴','绍兴','金华','舟山','台州']
K=['合肥','芜湖','马鞍山','铜陵','安庆','滁州','池州','宣城']
ua = UserAgent()
#D=['上海','南京','无锡','常州','苏州','南通','扬州','镇江','盐城','泰州','杭州','宁波','温州','湖州','嘉兴','绍兴','金华','舟山','台州','合肥','芜湖','马鞍山','铜陵','安庆','滁州','池州','宣城']
P=[]
B=[]
for n in L:C=[f for f in range(1,51)]E=[i for i in range(1,61)]h={'User-Agent':ua.random,'Cookie':'firstvisit_backurl=http%3A//www.wanfangdata.com.cn; curr=WFATT_MzM2QwZGRh; JSESSIONID=636EF84E82A62EC12C2387D31F8FAE21; Hm_lvt_838fbc4154ad87515435bf1e10023fab=1597042545,1597043396,1597132782,1597133289; SEARCHHISTORY_0=UEsDBBQACAgIAKulC1EAAAAAAAAAAAAAAAABAAAAMO2ceW8TRxTAv4ul5S8Ec%2B7MIKFq44Mk5I4T%0AJ6n6x8Ze2wv22l6vc7hCoioFFAkV1FKklh6qWmirtlRCLS3q8WEah%2FRb9K0DAoR3ISj2bkykSJnx%0AzK40v%2FfmzbzDfvvdhGeuVqwps2olTjmtSuV4wi4kTiUWZmbNjZo5RQlPHE%2B0mpY7VngyoWmZbr6c%0A3azDIxgG3Qo8UPa8%2BqmTJ9fX10%2Bsm07RdEoF0zNP5GvVE3nn5N4jj%2F9N2E3vRKH21tP3nK6bnuV4%0Ax5rl2nq3f6xulqx5u22d5ujY3rxczS2c1tJCU0wboVpaakZGGxFaGrqG%2F0cy3bbUFIxyTcIEpaV1%0ATaU0SZ9Oo0Z3NKWpJ88ayWN2M%2BvapZLlZs3SaViwtVF3rWbTrjmwtEcf39%2B992D74cOT279%2F1Ln6%0A%2Fc7n70PnVOfare2HP8Jkr0sisTcIfadVzdRaDvCiSHJdHU%2FkXQsWmLV9yJgrgZkSRAHbC8eDBDCZ%0Aa1cLcnQKE7YvATjWAQugD%2Fi706Thj45weMlr4d%2F%2BfWvn1weh%2BBVCnMge%2BLHiRADZYPxr3Cgy3nIo%0A1qPVf9oP%2FY9aAIhzhjmwDRZAa3WJ1w1VIypiA0TQkEqA6hTgBkug6G6sTZPRGYwilgAeVgFQDGyD%0ABdDwpDWfSgqMULQCGEr8ROkEyAbjn%2FXG245TnOFcRIG%2FYHmmXXnWDg2jGBgVCgPhYDHUVpzqWq1h%0AKElivQt6U4dGd%2FR1YT8m3fnwwatgRr0JIx3gBRP2Rmfr883sGEgi1oQPqYJzhIFsMP7SWGFq2Swa%0AlMlY44%2BvgjOdALxgwuNrdrmYaRUZoYMgnK85Rcu1nLz16tYcuAnNMHzsEjDqj7EbqW4DRpGP2mD%2B%0AqC8apEk9HO%2FulR8eXfpt995Pu99d7Hxz6dH1y8%2BQ%2FfPT3Z9%2Ffo4sAYeoB1wqJMHALRjuQnp5tVGq%0AS0lUHI%2FJAYN9UWUJuDo9weqSAbNgsItjlmVW1Tkw4AMBa7l2La5cd25%2F2bm69RxXXYpe1oAKQgUg%0AC7nXzUzWkFga4TKS4y4cbLMcDqlZDlc2QVRPKrrSKSw4mIpaq1ZtvmpxPJhtPCTapkslAFkw1%2BWc%0Aq7M2mmH7C%2BMdORH7uGNRnYMnzULCeUYzZ5tzXkXJeN%2BxDil%2BpiSQDTnKsJHc5LqMOJY6lPA5IRKh%0AkEiqVU7l8ig5zfhAbr9vGH6mmACywfgnVjOlRWKUuIp3%2FOKQ4peEAdlg%2FGO4nD5Lp5WK5gDuK%2F6o%0A05iIUgw3y7CD16wvnVfOdEYd2f4%2BaD9WjALZENt%2FPp0cn7d08O6P8B88fix9ssH4K5ueN7fkpsSA%0AnKo43P77Dh5cWakjLMK8rtT65NlGY1TSaELaLwP%2FMmavBAczLF6EIyWn0l93MJyV%2BhSfGLEmpB5D%0An3QsmQ3fyDAh%2FLCiivfEwgiYypCjSk5MsZXFc%2Bf0OMaFDvVmpUwPCz05Yi63bJoIy0is5LAfUgrr%0AQDYkQtVsnc03nJTiR%2Bm%2Fg8dPCAaDFKL9Z%2FHSipsRiwId3ZAPHj8VUoqw6IizdG46bUwaFB%2FdkPvh%0AnlMGZIPx2%2B3GZrk9UZLkKDDbh9ggxhzIBuOv8oV89fxGmpAj49OP0KwgQDYY%2F2jS2jjjLdngz8S1%0AMAEopjSV7KbgkCb5k6gTfzwE7f3D3v7r9u7FDzrXbm7%2Fde1FxD2KE5TO0YvOTjfzg3xvMBgxni9N%0AjrRndYZiFP8Lp7PvChiq60TAAoMpzEyW5qZXak0mBkLBrFSCNex10917Wdl%2FL77X%2BekO6A009jTF%0Ab3Q5QePR3Rs7Wxd3tr7t3Nh62r16feeL29Dd%2BepK58plv3H%2F5u7dS%2F6rbv%2By88kfnX9uQXv37uX%2F%0Avr7hf%2FjZ39t%2F3HleARHSBemdWOc6UA1Gv9Req89NtQpEjyT%2Bv99QxEs27msHJrqoBAIKIafRgr0w%0AieYW2WAcwYJVcq3Y1mjtaflzaIVEtHcpERcszMWbmG2slm2iCI1hxOegjaEQmMBCQ4KD07Mr9XHT%0Ao2IwDu%2BQlLoIqUtAFpJsyJouylX8W098jtrDVcgqEcALJrw6Ikcnk5spjGO4j4fl2s78YjkgHBIp%0AzpbcQr7CMIvkm1FvihgElkA4WAyFFh7nxdY8jiZ084aIgQkpcFgIx%2BGpvNFWZ%2BgQVlhEjp9zTHUa%0AVmFRXl5j%2BRKqyai%2Fpjmc%2Bs%2B5n70K%2B5pmbdmdJblkgaiI6%2BuGVQAMIWAbks9ueOs5e9Si0eSznwoA%0A0SEVAOM0LHO%2Bnq2kM8aYLWnEJmgIa%2BwwxYgLIBuMvzmXy6THFpcZ258BOvCfCunLb7VErf%2BYKT9u%0AwuiFd%2F4HUEsHCJAi2F%2FOBgAALkcAAA%3D%3D%0A; Hm_lpvt_838fbc4154ad87515435bf1e10023fab=1597202489; zh_choose=n'}for a in E:B.append('http://www.wanfangdata.com.cn/search/searchList.do?beetlansyId=aysnsearch&searchType=patent&pageSize=50&page='+str(a)+'&searchWord=申请人%2F专利权人%3A'+n+'&order=correlation&showType=detail&isCheck=check&isHit=&isHitUnit=&facetField=%24patent_type%3A发明专利&facetName=发明专利:')
G=[]
for b in B:c=requests.get(b,headers=h)d=etree.HTML(c.text)for e in C:if d.xpath('/html/body/div/div['+str(e)+']/div[2]/div[2]/div[1]/span[3]/text()')==[]:G.append('')else:G.append(d.xpath('/html/body/div/div['+str(e)+']/div[2]/div[2]/div[1]/span[3]/text()')[0])
H=[i for i in G if i !='']
len(H)

#h2={'User-Agent':ua.random,'Content-Type':'application/json','Cookies':'CASTGC=TGT-5225725-efSGlJRr0cAiFxjChaMZz42EengJed0fAnTTm4u5DRcAqdZV9Z-my.wanfangdata.com.cn;CASTGCSpecial=','Cookie':'zh_choose=n; Hm_lvt_838fbc4154ad87515435bf1e10023fab=1596885368; firstvisit_backurl=http%3A//www.wanfangdata.com.cn; SEARCHHISTORY_0=UEsDBBQACAgIACSaCFEAAAAAAAAAAAAAAAABAAAAMM2QzUrEMBSF3yWQrko7nf5MWihidcTBOhsL%0AguIittc20KadJKX%2BMHtx61P4AKIPpD6HmRlFEHXtKtzknNxzvtMbpOh5DXPaAIp4X9cmYgWK0O5w%0AeLBY7BM3dJCJegliVnwKJFCRV9lVpy2OfhS1NlRKdZFtD8NgDZRfUF4WVFErbxsr5%2FbG8nGkTCqr%0AaLe%2B%2Fok7qoArQ1btsJ4LUJTVRkdLOGLXEI9HxkZ93IoixlMPJwSTbTwNcOLjZKIzwmUnQErWcp3m%0A5fnu7fFJ36p1Sj3fv94%2B6Jn3zV7bc90lHDmeMzFRLkAvz9gKgOOHASG%2BS1a9l%2BZvcE66uZ8mkJLA%0A%2B39wZjuZwWQmWFmCyGgZf2ejBX%2BCcd3Q%2FxGLNw514%2BXZO1BLBwjMH0AtIAEAADUCAAA%3D%0A; CASTGC=TGT-2308888-jNwZ97RG4Y2U6T93MSnremgdoFr2ZPzyfpvZ5QLhlXGhYadu6B-my.wanfangdata.com.cn; WFKS.Auth=%7B%22Context%22%3A%7B%22AccountIds%22%3A%5B%22Person.WFATT_MzM2QwZGRh%22%5D%2C%22Data%22%3A%5B%7B%22Key%22%3A%22Person.WFATT_MzM2QwZGRh.DisplayName%22%2C%22Value%22%3A%22%22%7D%5D%2C%22SessionId%22%3A%22901b3cf8-b486-4ad8-9e99-fdc4d5580ca8%22%2C%22Sign%22%3A%22JTpKYCQUwUEDmphxnuKd4L2RexKEVgnqBOyFwlQzUV9WAEIoNSC8HGEKTFXcWDdV%22%7D%2C%22LastUpdate%22%3A%222020-08-08T11%3A20%3A21Z%22%2C%22TicketSign%22%3A%22bbxSRjH1lv2DEBc4RXCfew%3D%3D%22%2C%22UserIp%22%3Anull%7D; noReadCollection=0; curr=WFATT_MzM2QwZGRh; HISTORYCOMBINESEARCHES=%5B%5D; Hm_lpvt_838fbc4154ad87515435bf1e10023fab=1596887146'}
for k in Q:O=[]url='http://d.wanfangdata.com.cn/Detail/Patent'body={'Id':k}w=requests.post(url,data=json.dumps(body))data=json.loads(w.content.decode('utf_8'))time.sleep(random.uniform(0.006,0.01))O.append(data['detail'][0]['patent']['Title'][0])#专利名称O.append(data['detail'][0]['patent']['PatentCode'])#申请/专利号O.append('/'.join(data['detail'][0]['patent']['Applicant']))#申请/专利权人O.append(data['detail'][0]['patent']['Inventor'][0])#发明/设计人O.append(data['detail'][0]['patent']['ApplicantAddress'])#主申请人地址O.append(data['detail'][0]['patent']['ApplicantArea'])#国别省市代码O.append(data['detail'][0]['patent']['ApplicationDate'])#申请日期O.append(data['detail'][0]['patent']['PublicationDate'])#公开日期O.append(data['detail'][0]['patent']['PublicationNo'])#公开号P.append(O)
Z=pd.DataFrame(P)
Z.rename(columns={0:'专利名称',1:'申请/专利号',2:'申请/专利权人',3:'发明/设计人',4:'主申请人地址',5:'国别省市代码',6:'申请日期',7:'公开日期',8:'公开号'}, inplace = True)
#m='C:/Users/admin/Desktop/长江三角洲专利数据/81I.xlsx'
Z#.to_excel(m,encoding='utf_8')