#!/usr/bin/env python
# coding=utf-8

import numpy as np

import scrapy
from scrapy.selector import Selector
from douban.items import BookItem
import douban.pipelines as pipe

# 这个spider的作用是测试一些新加入的功能，它只抓取一本书
class TestSpider(scrapy.Spider):
    name = "douban.test"
    allowed_domains = ['douban.com']
    
    # bookid为crawl命令传递过来的参数
    def __init__(self, bookid=None, *args, **kwargs):
        super(TestSpider, self).__init__(*args, **kwargs)
        if bookid == None:
            bookid = '1082154'
        self.start_urls = ['https://book.douban.com/subject/%s' % bookid]

    pipeline = set([
       pipe.IDPipeline,
       pipe.BookInfoPipeline,
    ])

    # 抽取每本书籍，并将其存入对应的BookItem
    def parse(self, response):
        item = BookItem()
        sel = Selector(response)
        e = sel.xpath("//div[@id='wrapper']")
        item['name'] = e.xpath("./descendant::h1/descendant::span/text()").extract()
        item['author'] = e.xpath("//*[@id='info']/span[1]/a/text()").extract()
        item['bookinfo'] = e.xpath("//*[@id='info']/text()").extract()
        item['score'] = e.xpath('//*[@id="interest_sectl"]/div/div[2]/strong/text()').extract()
        item['commentNum'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@property = "v:votes"]/text()').extract()

        item['fivestar'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][1]/text()').extract()
        item['fourstar'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][2]/text()').extract()
        item['threestar'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][3]/text()').extract()
        item['twostar'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][4]/text()').extract()
        item['onestar'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][5]/text()').extract()

        item['tag'] = response.xpath("//*[@id = 'db-tags-section']/descendant::a/text()").extract()

        request = scrapy.Request(response.url + "/comments/hot", callback=self.parse_review) # 发送书籍对应的短评请求
        request.meta['item'] = item

        return request


    # 抽取书籍的热门短评信息
    def parse_review(self, response):
        item = response.meta['item']

        item['review'] = response.xpath("//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/p/text()").extract()

        item['reviewTime'] = response.xpath("//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/span[last()]/text()").extract()

        item['reviewName'] = response.xpath("//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/text()").extract()

        item['ID'] = response.xpath("//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/@href").extract()

        item['reviewHelpfulNum'] = response.xpath("//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-vote']/span/text()").extract()

        return item


84c22df9247b8cfd7a1bf0f2a85fde2472655daf
<mask token>


class TestSpider(scrapy.Spider):
    <mask token>
    <mask token>
    <mask token>
    <mask token>

    def parse(self, response):
        item = BookItem()
        sel = Selector(response)
        e = sel.xpath("//div[@id='wrapper']")
        item['name'] = e.xpath('./descendant::h1/descendant::span/text()'
            ).extract()
        item['author'] = e.xpath("//*[@id='info']/span[1]/a/text()").extract()
        item['bookinfo'] = e.xpath("//*[@id='info']/text()").extract()
        item['score'] = e.xpath(
            '//*[@id="interest_sectl"]/div/div[2]/strong/text()').extract()
        item['commentNum'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@property = "v:votes"]/text()'
            ).extract()
        item['fivestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][1]/text()'
            ).extract()
        item['fourstar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][2]/text()'
            ).extract()
        item['threestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][3]/text()'
            ).extract()
        item['twostar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][4]/text()'
            ).extract()
        item['onestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][5]/text()'
            ).extract()
        item['tag'] = response.xpath(
            "//*[@id = 'db-tags-section']/descendant::a/text()").extract()
        request = scrapy.Request(response.url + '/comments/hot', callback=
            self.parse_review)
        request.meta['item'] = item
        return request

    def parse_review(self, response):
        item = response.meta['item']
        item['review'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/p/text()"
            ).extract()
        item['reviewTime'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/span[last()]/text()"
            ).extract()
        item['reviewName'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/text()"
            ).extract()
        item['ID'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/@href"
            ).extract()
        item['reviewHelpfulNum'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-vote']/span/text()"
            ).extract()
        return item

<mask token>


class TestSpider(scrapy.Spider):
    <mask token>
    <mask token>

    def __init__(self, bookid=None, *args, **kwargs):
        super(TestSpider, self).__init__(*args, **kwargs)
        if bookid == None:
            bookid = '1082154'
        self.start_urls = ['https://book.douban.com/subject/%s' % bookid]
    <mask token>

    def parse(self, response):
        item = BookItem()
        sel = Selector(response)
        e = sel.xpath("//div[@id='wrapper']")
        item['name'] = e.xpath('./descendant::h1/descendant::span/text()'
            ).extract()
        item['author'] = e.xpath("//*[@id='info']/span[1]/a/text()").extract()
        item['bookinfo'] = e.xpath("//*[@id='info']/text()").extract()
        item['score'] = e.xpath(
            '//*[@id="interest_sectl"]/div/div[2]/strong/text()').extract()
        item['commentNum'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@property = "v:votes"]/text()'
            ).extract()
        item['fivestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][1]/text()'
            ).extract()
        item['fourstar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][2]/text()'
            ).extract()
        item['threestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][3]/text()'
            ).extract()
        item['twostar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][4]/text()'
            ).extract()
        item['onestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][5]/text()'
            ).extract()
        item['tag'] = response.xpath(
            "//*[@id = 'db-tags-section']/descendant::a/text()").extract()
        request = scrapy.Request(response.url + '/comments/hot', callback=
            self.parse_review)
        request.meta['item'] = item
        return request

    def parse_review(self, response):
        item = response.meta['item']
        item['review'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/p/text()"
            ).extract()
        item['reviewTime'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/span[last()]/text()"
            ).extract()
        item['reviewName'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/text()"
            ).extract()
        item['ID'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/@href"
            ).extract()
        item['reviewHelpfulNum'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-vote']/span/text()"
            ).extract()
        return item

<mask token>


class TestSpider(scrapy.Spider):
    name = 'douban.test'
    allowed_domains = ['douban.com']

    def __init__(self, bookid=None, *args, **kwargs):
        super(TestSpider, self).__init__(*args, **kwargs)
        if bookid == None:
            bookid = '1082154'
        self.start_urls = ['https://book.douban.com/subject/%s' % bookid]
    pipeline = set([pipe.IDPipeline, pipe.BookInfoPipeline])

    def parse(self, response):
        item = BookItem()
        sel = Selector(response)
        e = sel.xpath("//div[@id='wrapper']")
        item['name'] = e.xpath('./descendant::h1/descendant::span/text()'
            ).extract()
        item['author'] = e.xpath("//*[@id='info']/span[1]/a/text()").extract()
        item['bookinfo'] = e.xpath("//*[@id='info']/text()").extract()
        item['score'] = e.xpath(
            '//*[@id="interest_sectl"]/div/div[2]/strong/text()').extract()
        item['commentNum'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@property = "v:votes"]/text()'
            ).extract()
        item['fivestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][1]/text()'
            ).extract()
        item['fourstar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][2]/text()'
            ).extract()
        item['threestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][3]/text()'
            ).extract()
        item['twostar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][4]/text()'
            ).extract()
        item['onestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][5]/text()'
            ).extract()
        item['tag'] = response.xpath(
            "//*[@id = 'db-tags-section']/descendant::a/text()").extract()
        request = scrapy.Request(response.url + '/comments/hot', callback=
            self.parse_review)
        request.meta['item'] = item
        return request

    def parse_review(self, response):
        item = response.meta['item']
        item['review'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/p/text()"
            ).extract()
        item['reviewTime'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/span[last()]/text()"
            ).extract()
        item['reviewName'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/text()"
            ).extract()
        item['ID'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/@href"
            ).extract()
        item['reviewHelpfulNum'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-vote']/span/text()"
            ).extract()
        return item

import numpy as np
import scrapy
from scrapy.selector import Selector
from douban.items import BookItem
import douban.pipelines as pipe


class TestSpider(scrapy.Spider):
    name = 'douban.test'
    allowed_domains = ['douban.com']

    def __init__(self, bookid=None, *args, **kwargs):
        super(TestSpider, self).__init__(*args, **kwargs)
        if bookid == None:
            bookid = '1082154'
        self.start_urls = ['https://book.douban.com/subject/%s' % bookid]
    pipeline = set([pipe.IDPipeline, pipe.BookInfoPipeline])

    def parse(self, response):
        item = BookItem()
        sel = Selector(response)
        e = sel.xpath("//div[@id='wrapper']")
        item['name'] = e.xpath('./descendant::h1/descendant::span/text()'
            ).extract()
        item['author'] = e.xpath("//*[@id='info']/span[1]/a/text()").extract()
        item['bookinfo'] = e.xpath("//*[@id='info']/text()").extract()
        item['score'] = e.xpath(
            '//*[@id="interest_sectl"]/div/div[2]/strong/text()').extract()
        item['commentNum'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@property = "v:votes"]/text()'
            ).extract()
        item['fivestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][1]/text()'
            ).extract()
        item['fourstar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][2]/text()'
            ).extract()
        item['threestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][3]/text()'
            ).extract()
        item['twostar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][4]/text()'
            ).extract()
        item['onestar'] = e.xpath(
            '//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][5]/text()'
            ).extract()
        item['tag'] = response.xpath(
            "//*[@id = 'db-tags-section']/descendant::a/text()").extract()
        request = scrapy.Request(response.url + '/comments/hot', callback=
            self.parse_review)
        request.meta['item'] = item
        return request

    def parse_review(self, response):
        item = response.meta['item']
        item['review'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/p/text()"
            ).extract()
        item['reviewTime'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/span[last()]/text()"
            ).extract()
        item['reviewName'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/text()"
            ).extract()
        item['ID'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/@href"
            ).extract()
        item['reviewHelpfulNum'] = response.xpath(
            "//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-vote']/span/text()"
            ).extract()
        return item

#!/usr/bin/env python
# coding=utf-8

import numpy as np

import scrapy
from scrapy.selector import Selector
from douban.items import BookItem
import douban.pipelines as pipe

# 这个spider的作用是测试一些新加入的功能，它只抓取一本书
class TestSpider(scrapy.Spider):
    name = "douban.test"
    allowed_domains = ['douban.com']
    
    # bookid为crawl命令传递过来的参数
    def __init__(self, bookid=None, *args, **kwargs):
        super(TestSpider, self).__init__(*args, **kwargs)
        if bookid == None:
            bookid = '1082154'
        self.start_urls = ['https://book.douban.com/subject/%s' % bookid]

    pipeline = set([
       pipe.IDPipeline,
       pipe.BookInfoPipeline,
    ])

    # 抽取每本书籍，并将其存入对应的BookItem
    def parse(self, response):
        item = BookItem()
        sel = Selector(response)
        e = sel.xpath("//div[@id='wrapper']")
        item['name'] = e.xpath("./descendant::h1/descendant::span/text()").extract()
        item['author'] = e.xpath("//*[@id='info']/span[1]/a/text()").extract()
        item['bookinfo'] = e.xpath("//*[@id='info']/text()").extract()
        item['score'] = e.xpath('//*[@id="interest_sectl"]/div/div[2]/strong/text()').extract()
        item['commentNum'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@property = "v:votes"]/text()').extract()

        item['fivestar'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][1]/text()').extract()
        item['fourstar'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][2]/text()').extract()
        item['threestar'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][3]/text()').extract()
        item['twostar'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][4]/text()').extract()
        item['onestar'] = e.xpath('//*[@id="interest_sectl"]/descendant::span[@class = "rating_per"][5]/text()').extract()

        item['tag'] = response.xpath("//*[@id = 'db-tags-section']/descendant::a/text()").extract()

        request = scrapy.Request(response.url + "/comments/hot", callback=self.parse_review) # 发送书籍对应的短评请求
        request.meta['item'] = item

        return request


    # 抽取书籍的热门短评信息
    def parse_review(self, response):
        item = response.meta['item']

        item['review'] = response.xpath("//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/p/text()").extract()

        item['reviewTime'] = response.xpath("//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/span[last()]/text()").extract()

        item['reviewName'] = response.xpath("//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/text()").extract()

        item['ID'] = response.xpath("//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-info']/a/@href").extract()

        item['reviewHelpfulNum'] = response.xpath("//*[@id = 'comment-list-wrapper']/descendant::li[@class = 'comment-item'][1]/descendant::span[@class='comment-vote']/span/text()").extract()

        return item

