华为云照片的爬虫程序更新(python3.6)

'''

try :

#loginConfirm = self.SeleBrowser.find_element_by_class_name("global_dialog_confirm_main")

loginConfirm =WebDriverWait(self.SeleBrowser, 5, 0.5).until(EC.presence_of_element_located((By.CLASS_NAME, 'global_dialog_confirm_main') ))

#需要验证，这块懒得实现了，休眠60秒，手动操作吧。

if loginConfirm.is_displayed():

time.sleep(self.TimeOUT*2)

except:

#不需要验证，直接下一步

pass

#等待页面加载完毕

'''

图库

'''

try :

#loginConfirm = self.SeleBrowser.find_element_by_class_name("global_dialog_confirm_main")

success =WebDriverWait(self.SeleBrowser, 20, 0.5).until(EC.presence_of_element_located((By.XPATH, '//span[@data-bind="lang.common.album"]') ))

except Exception as e:

#登录失败

return self.QuitSele(e,"登录失败！",iRet=-999)

#判断登录结果

if not success.is_displayed(): return self.QuitSele(None,"登录失败！",iRet=-999)

#再次判断，增加一次意外处理

source_code =self.SeleBrowser.page_source

if '联系人' not in source_code or '图库' not in source_code :

return self.QuitSele(None,"登录失败！",iRet = -9999 )

cookie = [item["name"] + "=" + item["value"] for item in self.SeleBrowser.get_cookies()]

cookiestr = ';'.join(item for item in cookie)

#保存CSRFToken

pattern = re.compile('CSRFToken = "(.*?)"',re.S)

content = re.search(pattern,source_code)

if content :

CSRFToken = content.group(1)

else :

print ('获取CSRFToken出错！')

self.Headers={

'User-Agent': '%s' %self.Headers,

'CSRFToken': '%s' %CSRFToken,

'Cookie': '%s' %cookiestr

}

return 1

if __name__ == '__main__':

photohw= HuaWei()

count =0

while (count <100):

count += 1

selehw= hwSele(SeleDriver='edge')

iRet = selehw.LoginHW()

if iRet !=1:

print( '登录华为失败！！！\n\n')

continue

photohw.loginHeaders = selehw.Headers

page = photohw.getAlbumList()

if page=='' :

print( '获取到相册列表失败！！！\n\n')

break

#保存相册列表

iRet = photohw.getFileList(page,'albumList','albumId')

if iRet <=0 :

print('保存相册出错，重新登录')

continue

#保存公共相册列表

iRet = photohw.getFileList(page,'ownShareList','shareId')

if iRet ==0 :

print('运行结束，可以用迅雷打开相册文件进行批量下载到本地！！！\n\n')

#运行结束

selehw.QuitSele(None)

break

else:

continue

huaweiphoto_py3.py如下：

# -*- coding=utf-8 -*-

# Create by : zhongtang

# Create date : 2018.2.28

import json

import requests

from requests.adapters import HTTPAdapter

import html

class HuaWei:

#华为云服务登录

def __init__(self):

self.getalbumsUrl= 'https://www.hicloud.com/album/getCloudAlbums.action'

self.getalbumfileUrl = 'https://www.hicloud.com/album/getCloudFiles.action'

self.loginHeaders = { }

self.SReq=requests.session()

self.SReq.mount('http://', HTTPAdapter(max_retries=3))

self.SReq.mount('https://', HTTPAdapter(max_retries=3))

self.OnceMaxFile=100 #单次最大获取文件数量

self.FileNum=0

self.AlbumList={}

#保存相册照片地址到文件 ,不同相册保存到不同的文件

def saveFileList2Txt(self,filename,hjsondata,flag):

if len(hjsondata)<= 0 : return -1

hjson2 = {}

try:

hjson2 = json.loads(hjsondata)

except:

print('获取相册明细出错\n')

return -1

lfilename = filename+u".txt"

if flag == 0 : #新建文件

print( u'创建相册文件'+lfilename+"\n")

#新建文件，代表新的相册重新开始计数

self.FileNum = 0

f = open(lfilename, 'w')

else: #追加文件

f = open(lfilename, 'a')

i = 0

if hjson2.get("fileList"):

for each in hjson2["fileList"]:

fileurl= html.unescape(hjson2["fileList"][i]["fileUrl"])

f.write(fileurl+"\n")

#每一千行分页

self.FileNum += 1

if self.FileNum%1000 ==0 :f.write('\n\n\n\n\n\n--------------------page %s ------------------\n\n\n\n\n\n' %(int(self.FileNum/1000)))

i += 1

f.close()

return i

#循环读取相册文件

def getFileList(self,hjsondata,parentkey,childkey):

#step 3 getCoverFiles.action,循环取相册文件列表，单次最多取100条记录。

#每次count都是最大数量49，不管实际数量是否够，每次currentnum递增，直到返回空列表。

#albumIds[]=default-album-2&ownerId=220086000029851117&height=300&width=300&count=49¤tNum=0&thumbType=imgcropa&fileType=0

#albumIds[]=default-album-1&ownerId=220086000029851117&height=300&width=300&count=49¤tNum=49&thumbType=imgcropa&fileType=0

#albumIds[]=default-album-1&ownerId=220086000029851117&height=300&width=300&count=49¤tNum=98&thumbType=imgcropa&fileType=0

#albumIds[]=default-album-2&ownerId=220086000029851117&height=300&width=300&count=49¤tNum=101&thumbType=imgcropa&fileType=0

#最后一次返回 空列表

#{"albumSortFlag":true,"code":0,"info":"success!","fileList":[]}

#第一次取文件时，例如文件总数量只有2个，count也是放最大值49。

#albumIds[]=default-album-102-220086000029851117&ownerId=220086000029851117&height=300&width=300&count=49¤tNum=0&thumbType=imgcropa&fileType=0

#[{u'photoNum': 2518, u'albumName': u'default-album-1', u'iversion': -1, u'albumId': u'default-album-1', u'flversion': -1, u'createTime': 1448065264550L, u'size': 0},

#{u'photoNum': 100, u'albumName': u'default-album-2', u'iversion': -1, u'albumId': u'default-album-2', u'flversion': -1, u'createTime': 1453090781646L, u'size': 0}]

try:

hjson = json.loads(hjsondata)

except Exception:

print ('加载json出错!')

return -1

#字典获取出错

if not hjson.get(parentkey):

print ('加载json根节点[%s]出错!' %parentkey)

return -1

#初始化全局 albumlist

if not self.AlbumList :

self.AlbumList=hjson

for idx,album in enumerate(self.AlbumList[parentkey]):

if 'currentNum' not in self.AlbumList[parentkey][idx].keys():

self.AlbumList[parentkey][idx]['currentNum']=0

#循环保存相册

for each in hjson[parentkey]:

#该相册已经进入记录

paraAlbum={}

paraAlbum['albumIds[]'] = each[childkey]

paraAlbum['ownerId'] = hjson['ownerId']

paraAlbum['height'] = '300'

paraAlbum['width'] = '300'

paraAlbum['count'] = self.OnceMaxFile

paraAlbum['thumbType'] = 'imgcropa'

paraAlbum['fileType'] = '0'

itotal= each['photoNum']

#取当前节点的当前记录

for idx,album in enumerate(self.AlbumList[parentkey]):

if each[childkey]==album[childkey]:

icurrentnum = self.AlbumList[parentkey][idx]['currentNum']

break

#保存相册中所有文件

while icurrentnum

paraAlbum['currentNum'] = icurrentnum

response=self.SReq.post(self.getalbumfileUrl,headers=self.loginHeaders,data=paraAlbum,verify=False)

page = response.text

#保存下载地址到文本文件中，但不下载文件

iret = self.saveFileList2Txt(each[childkey],page,icurrentnum)

if iret >0 :

self.AlbumList[parentkey][idx]['currentNum'] += iret

icurrentnum = self.AlbumList[parentkey][idx]['currentNum']

else:

#出错！！！

return -1

return 1

#step 1 getCloudAlbums,取相册列表

def getAlbumList(self):

response=self.SReq.post(self.getalbumsUrl,headers=self.loginHeaders,verify=False)

page=response.text

'''#返回报文

{"ownerId":"220086000029851117","code":0,

"albumList":[{"albumId":"default-album-1","albumName":"default-album-1","createTime":1448065264550,"photoNum":2521,"flversion":-1,"iversion":-1,"size":0},

{"albumId":"default-album-2","albumName":"default-album-2","createTime":1453090781646,"photoNum":101,"flversion":-1,"iversion":-1,"size":0}],

"ownShareList":[{"ownerId":"220086000029851117","resource":"album","shareId":"default-album-102-220086000029851117","shareName":"微信","photoNum":2,"flversion":-1,"iversion":-1,"createTime":1448070407055,"source":"HUAWEI MT7-TL00","size":0,"ownerAcc":"****","receiverList":[]}],

"recShareList":[]}'

'''

if len(page)<=0 :

print( u'取相册列表出错，无返回报文!!!\n\n')

return page

五、运行结果：

程序会在当前目录生成华为云相册照片下载地址文件，内容如下：

https://d167.g03.dbankcloud.com/file/MDAwMTZBODissQaaaaaaaaaaaaaaaaaaaaQc2CR-znjyRnw../162807b277aaaaaaaaaaaaaaaaaa9ee1/IMG_20170606_141952.jpg?key=AWqIQFqVkEaaaaaaaaaaaaaaaaaaaaaaWNLIosPR_EKv8VQ..&a=220086000029851117-3da1ab76-92808-5840&nsp_ver=3.0https://d167.g03.dbankcloud.com/file/MDAwMTZBODhhoFaaaaaaaaaaaaaaaaaaaa7r6jPU67bWTQA../4039a1be5caaaaaaaaaaaaaaaaaac726/IMG_20170605_203519.jpg?key=AWqIQFqVkEaaaaaaaaaaaaaaaaaaaaaajvduIL8cXufhNhQ..&a=220086000029851117-3da1ab76-92808-5840&nsp_ver=3.0https://d167.g03.dbankcloud.com/file/MDAwMTZBODgpoWaaaaaaaaaaaaaaaaaaaaaciAQlIVHRbXg../9e336da286aaaaaaaaaaaaaaaaaaf89d/IMG_20170604_171032.jpg?key=AWqIQFqVkEaaaaaaaaaaaaaaaaaaaaaaThkDgJKHpBtiG5w..&a=220086000029851117-3da1ab76-92808-5840&nsp_ver=3.0https://d167.g03.dbankcloud.com/file/MDAwMTZBODgUz2aaaaaaaaaaaaaaaaaaaatyFMDr71YpXGg../b3c17582ccaaaaaaaaaaaaaaaaac278b/IMG_20170603_134831.jpg?key=AWqIQFqVkEaaaaaaaaaaaaaaaaaaaaaaTI_xPSzF_VzUsJA..&a=220086000029851117-3da1ab76-92808-5840&nsp_ver=3.0https://d167.g03.dbankcloud.com/file/MDAwMTZBODgGfwaaaaaaaaaaaaaaaaaaaad7DLSHH4rwKVA../2722df087baaaaaaaaaaaaaaaaa915e4/IMG_20170603_133833.jpg?key=AWqIQFqVkEaaaaaaaaaaaaaaaaaaaaaaluTQ8grDHok9BzQ..&a=220086000029851117-3da1ab76-92808-5840&nsp_ver=3.0https://d167.g03.dbankcloud.com/file/MDAwMTZBODgsrHaaaaaaaaaaaaaaaaaaaatq2yOJ-OnkDtQ../77e0ef0560aaaaaaaaaaaaaaaaa44702/IMG_20170602_183736.jpg?key=AWqIQFqVkEaaaaaaaaaaaaaaaaaaaaaa20WJbfxn-qoqIeQ..&a=220086000029851117-3da1ab76-92808-5840&nsp_ver=3.0https://d167.g03.dbankcloud.com/file/MDAwMTZBODiAcDaaaaaaaaaaaaaaaaaaaaEXW-ONoF0Shuw../df033e69ffaaaaaaaaaaaaaaaaa8c1b1/IMG_20170601_185446.jpg?key=AWqIQFqVkEaaaaaaaaaaaaaaaaaaaaaaxIC-spleDG_xxVg..&a=220086000029851117-3da1ab76-92808-5840&nsp_ver=3.0https://d167.g03.dbankcloud.com/file/MDAwMTZBODjY8EaaaaaaaaaaaaaaaaaaaaVbr9kC-JU0M8g../d5230d2032aaaaaaaaaaaaaaaaa903b3/IMG_20170601_102059.jpg?key=AWqIQFqVkEaaaaaaaaaaaaaaaaaaaaaa49iM03bK-Cm-Z9g..&a=220086000029851117-3da1ab76-92808-5840&nsp_ver=3.0https://d167.g03.dbankcloud.com/file/MDAwMTZBODi04Caaaaaaaaaaaaaaaaaaaaaaw41bSNB4pxBw../6ee510e28aaaaaaaaaaaaaaaaa57cb5a/IMG_20170601_102042.jpg?key=AWqIQFqVkaaaaaaaaaaaaaaaaaaaaaaxlapdsHLoRCSITVw..&a=220086000029851117-3da1ab76-92808-5840&nsp_ver=3.0

把上述下载链接复制到迅雷，添加批量任务就可以下载图片到本地。

以上，-- End -- 

查看原文