本文共 5555 字,大约阅读时间需要 18 分钟。
简单使用
import threading,time'''想要学习Python?Python学习交流群:973783996满足你的需求,资料都已经上传群文件,可以自行下载!'''def coding(): for x in range(3): print('正在写代码%s'%x) time.sleep(2)def drawing(): for x in range(3): print('正在画画%s'%x) time.sleep(2)def main(): t1 = threading.Thread(target=coding) t2 = threading.Thread(target=drawing) t1.start() t2.start()if __name__ == '__main__': main()
Lock模式的生产者和消费者
import threadingimport random,timegMoney = 1000gLock = threading.Lock()gTotalTimes = 10gTimes = 0class Producer(threading.Thread): def run(self): global gMoney global gTimes while True: money = random.randint(100,1000) gLock.acquire() #只生产10次,超过就停止,必须把锁给释放掉,否则产生死锁 if gTimes >= gTotalTimes: gLock.release() break gMoney += money print('%s生产了%d元钱,剩余%d元钱' % (threading.current_thread(), money, gMoney)) #生产一次,次数加1,总共10次 gTimes += 1 gLock.release() time.sleep(0.5)class Consumer(threading.Thread): def run(self): global gMoney while True: money = random.randint(100,1000) gLock.acquire() if gMoney >= money: gMoney -= money print('%s消费了%d元钱,剩余%d元钱' % (threading.current_thread(), money,gMoney)) else: if gTimes >= gTotalTimes: gLock.release() break gLock.release() time.sleep(0.5)def main(): for x in range(5): t1 = Producer() t1.start() for x in range(2): t2 = Consumer() t2.start()if __name__ == '__main__': main()
网址:http://www.doutula.com/photo/list/?page=1
解析:xpath
不用多线程,速度相对会很慢
import requestsfrom lxml import etreefrom urllib import requestimport osimport re'''想要学习Python?Python学习交流群:973783996满足你的需求,资料都已经上传群文件,可以自行下载!'''def parse_page(url): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36', 'Referer': 'https://movie.douban.com/' } response = requests.get(url,headers=headers) text = response.text html = etree.HTML(text) imgs = html.xpath("//div[@class='page-content text-center']//img[@class!='gif']") for img in imgs: # print(etree.tostring(img)) #图片地址 img_url = img.get('data-original') #图片名字 alt = img.get('alt') #替换掉名字里面的特殊字符 alt = re.sub(r'[\??\.,。!!\*]','',alt) #获取图片的后缀名(.gif .jpg) suffix = os.path.splitext(img_url)[1] #保存的时候完整的图片名字 filename = alt + suffix request.urlretrieve(img_url,'C:/Users/Administrator/Desktop/images/'+filename)def main(): for x in range(1,10): url = 'http://www.doutula.com/photo/list/?page=%d'%x parse_page(url)if __name__ == '__main__': main()
利用多线程
main()
Producer()
Consumer()
代码
import requestsfrom lxml import etreefrom urllib import requestimport osimport reimport threadingfrom queue import Queue'''想要学习Python?Python学习交流群:973783996满足你的需求,资料都已经上传群文件,可以自行下载!'''class Producer(threading.Thread): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36', 'Referer': 'https://movie.douban.com/' } def __init__(self, page_queue, img_queue, *args, **kwargs): super(Producer, self).__init__(*args, **kwargs) self.page_queue = page_queue self.img_queue = img_queue def run(self): while True: if self.page_queue.empty(): break url = self.page_queue.get() self.parse_page(url) def parse_page(self,url): response = requests.get(url,headers=self.headers) text = response.text html = etree.HTML(text) imgs = html.xpath("//div[@class='page-content text-center']//img[@class!='gif']") for img in imgs: # print(etree.tostring(img)) #图片地址 img_url = img.get('data-original') #图片名字 alt = img.get('alt') #替换掉名字里面的特殊字符 alt = re.sub(r'[\??\.,。!!\*]','',alt) #获取图片的后缀名(.gif .jpg) suffix = os.path.splitext(img_url)[1] #保存的时候完整的图片名字 filename = alt + suffix self.img_queue.put((img_url,filename))class Consumer(threading.Thread): def __init__(self,page_queue,img_queue,*args,**kwargs): super(Consumer, self).__init__(*args,**kwargs) self.page_queue = page_queue self.img_queue = img_queue def run(self): while True: if self.img_queue.empty() and self.page_queue.empty(): break img_url,filename = self.img_queue.get() request.urlretrieve(img_url, 'C:/Users/Administrator/Desktop/images/' + filename) print("已下载完一张图片")def main(): page_queue = Queue(1000) img_queue = Queue(10000) for x in range(1,1758): url = 'http://www.doutula.com/photo/list/?page=%d'%x page_queue.put(url) for x in range(10): t = Producer(page_queue,img_queue) t.start() for x in range(10): t = Consumer(page_queue,img_queue) t.start()if __name__ == '__main__': main()
结果:
转载地址:http://pinmf.baihongyu.com/