首頁>技術>

1. items.py

class DouyuspiderItem(scrapy.Item): name = scrapy.Field()# 儲存照片的名字 imagesUrls = scrapy.Field()# 照片的url路徑 imagesPath = scrapy.Field()# 照片儲存在本地的路徑

2. spiders/douyu.py

import scrapyimport jsonfrom douyuSpider.items import DouyuspiderItemclass DouyuSpider(scrapy.Spider): name = "douyu" allowd_domains = ["http://capi.douyucdn.cn"] offset = 0 url = "/file/2019/11/30/20191130171535_23969.jpg start_urls = [url + str(offset)] def parse(self, response): # 返回從json裡獲取 data段資料集合 data = json.loads(response.text)["data"] for each in data: item = DouyuspiderItem() item["name"] = each["nickname"] item["imagesUrls"] = each["vertical_src"] yield item self.offset += 20 yield scrapy.Request(self.url + str(self.offset), callback = self.parse)

3. 設定setting.py

ITEM_PIPELINES = {'douyuSpider.pipelines.ImagesPipeline': 1}# Images 的存放位置,之後會在pipelines.py裡呼叫IMAGES_STORE = "/Users/Power/lesson_python/douyuSpider/Images"# user-agentUSER_AGENT = 'DYZB/2.290 (iPhone; iOS 9.3.4; Scale/2.00)'

4. pipelines.py

import scrapyimport osfrom scrapy.pipelines.images import ImagesPipelinefrom scrapy.utils.project import get_project_settingsclass ImagesPipeline(ImagesPipeline): IMAGES_STORE = get_project_settings().get("IMAGES_STORE") def get_media_requests(self, item, info): image_url = item["imagesUrls"] yield scrapy.Request(image_url) def item_completed(self, results, item, info): # 固定寫法,獲取圖片路徑,同時判斷這個路徑是否正確,如果正確,就放到 image_path裡,ImagesPipeline原始碼剖析可見 image_path = [x["path"] for ok, x in results if ok] os.rename(self.IMAGES_STORE + "/" + image_path[0], self.IMAGES_STORE + "/" + item["name"] + ".jpg") item["imagesPath"] = self.IMAGES_STORE + "/" + item["name"] return item#get_media_requests的作用就是為每一個圖片連結生成一個Request物件,這個方法的輸出將作為item_completed的輸入中的results,results是一個元組,每個元組包括(success, imageinfoorfailure)。如果success=true,imageinfoor_failure是一個字典,包括url/path/checksum三個key。

在專案根目錄下新建main.py檔案,用於除錯

from scrapy import cmdlinecmdline.execute('scrapy crawl douyu'.split())

執行程式

py2 main.py

  • BSA-TRITC(10mg/ml) TRITC-BSA 牛血清白蛋白改性標記羅丹明
  • Java 中的 ClassLoader 載入機制