摘要：爬蟲練手，主要運用，由于要對內部進行分析，所以就直接用了正則匹配，平時也可以用用，應該更加方便思路讀取首頁，就是如為了全部抓取，我們這里都是結尾找到頁面里面的標簽里面的，這個這個地址就是頁面自動播放的下個頁面同時讀出即圖片地址，然后保存我這

爬蟲練手，主要運用requests，由于要對script內部進行分析，所以就直接用了 re 正則匹配，平時也可以用用Beautifulsoup， 應該更加方便

  
讀取首頁，就是如 http://newcar.xcar.com.cn/2674/2015/detail/1.htm；為了全部抓取，我們這里都是 1.htm 結尾

  
找到頁面里面的 script 標簽里面的 " var nextUrl " ，這個這個地址就是頁面自動播放的下個頁面;同時讀出 img src 即 圖片地址，然后保存（我這里是把 說明 等內容全部放進 目標文件的名稱中了，其實大家只要圖片的話完全不需要）

  
遞歸抓取 全部頁面

這里另外做的一點實際 弄了個xcar_lst 記錄所有頁面、圖片等信息，只是留作記錄，暫時沒用

python
# coding:utf-8



__author__ = "BONFY CHEN"



import requests

import re



PROXIES = None



HEADERS = {

    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36"

    , "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"

    , "Accept-Encoding": "gzip,deflate,sdch"

    , "Accept-Language": "zh-CN,zh;q=0.8"

}



BASE_FOLDER = "D:/xxx_folder/"





class xcarDown(object):



    _base_folder = None

    _proxies = None

    _headers = None

    _website = "http://newcar.xcar.com.cn"

    _xcar_lst = []



    def set_base_folder(self, base_folder):

        self._base_folder = base_folder



    def set_headers(self, headers):

        self._headers = headers



    def set_proxies(self, proxies):

        self._proxies = proxies



    def __init__(self, base_folder=BASE_FOLDER, proxies=PROXIES, headers=HEADERS):

        self.set_base_folder(base_folder)

        self.set_headers(headers)

        self.set_proxies(proxies)



    def download_image_from_url(self, url, name=None):



        """

        download_image_from_url

        :param url: the resource image url

        :param name: he destination file name

        :return:

        """



        local_filename = name + "_" + url.split("/")[-1]

        r = requests.get(url, proxies=self._proxies, headers=self._headers, stream=True)

        with open(self._base_folder + local_filename, "wb") as f:

            for chunk in r.iter_content(chunk_size=1024):

                if chunk:

                    f.write(chunk)

                    f.flush()

            f.close()

        return local_filename





    def download_xcar(self, url):



        """



        :param url: the source url in xcar.com.cn

                    http://newcar.xcar.com.cn/2674/2015/detail/1.htm

        :return:

        """



        r = requests.get(url, proxies=self._proxies, headers=self._headers)

        # print r.encoding

        r.encoding = "gbk"



        m1 = re.search(r"var nextUrl = "(?P.*.htm)"", r.text)

        next_url = m1.groupdict()["n_url"] if m1 else None



        m2 = re.search(r"
(?P", r.text)

        model = m5.groupdict()["model"] if m5 else ""



        if pic_url:

            try:

                self.download_image_from_url(pic_url, name="_".join([model, title, cont]))

                print "download complete: pic from {} ".format(pic_url)

            except IOError:

                print "file name IOERROR"

                self.download_image_from_url(pic_url, name=model)

                print "download complete: pic from {} ".format(pic_url)

            except Exception as e:

                print e



        dct = dict(pic_url=pic_url, next_url=next_url, title=title, cont=cont, model=model)

        self._xcar_lst.append(dct)

       if next_url[-4:] == ".htm":

            self.download_xcar(self._website + next_url)





if __name__ == "__main__":

    print("Welcome to the Pic Download for xcar.com")

    print("Downloaded files in the folder: " + BASE_FOLDER )

    print("---------------------------------------")



    id_modell = int(input("Please enter the modell id(eg.2674): "))

    year = int(input("Please enter the year (eg.2015): "))



    url = "http://newcar.xcar.com.cn/{}/{}/detail/1.htm".format(id_modell, year)

    xcar = xcarDown()

    xcar.download_xcar(url)