
from .fields import ComposeField
from ..parser.fields import ParserFields, ParserField

class ParserCompose( ComposeField, ParserField ):

	_fields = None

	def __init__( self, spider = None, **kwargs ):
		super( ParserCompose, self ).__init__( **kwargs )

		self.spider = spider
		self.composed = self.get_endpoints

	@property
	def fields( self ):
		if not self._fields:
			self._fields = [
				ParserFields.fields[ v ]( key = k, value = v )
				for k, v in self.value.items()
			]
		return self._fields

	@property
	def has_endpoints( self ):
		return bool( self.meta )

	def get_context( self, response ):
		ctx = {}
		for field in self.fields:
			ctx.update( field.get_context( response ) )
		return ctx

	def get_endpoints( self, *response ):

		if not self.has_endpoints:
			yield None
			return

		from scrapy import Request

		response = response[-1]
		meta = self.meta
		spider = self.spider

		selector = self.get_selector( response )

		if "item" in meta:
			from scrapy_compose.utils.load import resource as load_resource
			yield load_resource( meta[ "item" ] )( **self.get_context( response ) )

		if "items" in meta:
			for query, callback in meta[ "items" ].items():
				f_callback = getattr( spider, callback )
				if query.startswith( "@" ):
					for block in selector( query[1:] ):
						yield next( f_callback( block ) )

		if "requests" in meta:
			for url_str, callback in meta[ "requests" ].items():
				f_callback = getattr( spider, callback )

				if url_str.startswith( "@" ):
					# realize will return string data directly if len( selected ) is 1
					urls = selector( url_str[1:] ).extract()
				else:
					urls = [ url_str ]

				for url in urls:
					yield Request( url, callback = f_callback )


		if "follows" in meta:
			from scrapy.utils.response import get_base_url
			from urllib.parse import urljoin

			base_url = get_base_url( response )
			for url_str, callback in meta[ "follows" ].items():
				f_callback = getattr( spider, callback )

				if url_str.startswith( "@" ):
					# realize will return string data directly if len( selected ) is 1
					urls = [ urljoin( base_url, url ) for url in selector( url_str[1:] ).extract() ]
				else:
					urls = [ urljoin( base_url, url_str ) ]

				for url in urls:
					yield Request( url, callback = f_callback )

bc6eda054fc2d38cdce1364f44c1979dfd8e700f
<mask token>


class ParserCompose(ComposeField, ParserField):
    <mask token>
    <mask token>

    @property
    def fields(self):
        if not self._fields:
            self._fields = [ParserFields.fields[v](key=k, value=v) for k, v in
                self.value.items()]
        return self._fields

    @property
    def has_endpoints(self):
        return bool(self.meta)

    def get_context(self, response):
        ctx = {}
        for field in self.fields:
            ctx.update(field.get_context(response))
        return ctx

    def get_endpoints(self, *response):
        if not self.has_endpoints:
            yield None
            return
        from scrapy import Request
        response = response[-1]
        meta = self.meta
        spider = self.spider
        selector = self.get_selector(response)
        if 'item' in meta:
            from scrapy_compose.utils.load import resource as load_resource
            yield load_resource(meta['item'])(**self.get_context(response))
        if 'items' in meta:
            for query, callback in meta['items'].items():
                f_callback = getattr(spider, callback)
                if query.startswith('@'):
                    for block in selector(query[1:]):
                        yield next(f_callback(block))
        if 'requests' in meta:
            for url_str, callback in meta['requests'].items():
                f_callback = getattr(spider, callback)
                if url_str.startswith('@'):
                    urls = selector(url_str[1:]).extract()
                else:
                    urls = [url_str]
                for url in urls:
                    yield Request(url, callback=f_callback)
        if 'follows' in meta:
            from scrapy.utils.response import get_base_url
            from urllib.parse import urljoin
            base_url = get_base_url(response)
            for url_str, callback in meta['follows'].items():
                f_callback = getattr(spider, callback)
                if url_str.startswith('@'):
                    urls = [urljoin(base_url, url) for url in selector(
                        url_str[1:]).extract()]
                else:
                    urls = [urljoin(base_url, url_str)]
                for url in urls:
                    yield Request(url, callback=f_callback)

<mask token>


class ParserCompose(ComposeField, ParserField):
    <mask token>

    def __init__(self, spider=None, **kwargs):
        super(ParserCompose, self).__init__(**kwargs)
        self.spider = spider
        self.composed = self.get_endpoints

    @property
    def fields(self):
        if not self._fields:
            self._fields = [ParserFields.fields[v](key=k, value=v) for k, v in
                self.value.items()]
        return self._fields

    @property
    def has_endpoints(self):
        return bool(self.meta)

    def get_context(self, response):
        ctx = {}
        for field in self.fields:
            ctx.update(field.get_context(response))
        return ctx

    def get_endpoints(self, *response):
        if not self.has_endpoints:
            yield None
            return
        from scrapy import Request
        response = response[-1]
        meta = self.meta
        spider = self.spider
        selector = self.get_selector(response)
        if 'item' in meta:
            from scrapy_compose.utils.load import resource as load_resource
            yield load_resource(meta['item'])(**self.get_context(response))
        if 'items' in meta:
            for query, callback in meta['items'].items():
                f_callback = getattr(spider, callback)
                if query.startswith('@'):
                    for block in selector(query[1:]):
                        yield next(f_callback(block))
        if 'requests' in meta:
            for url_str, callback in meta['requests'].items():
                f_callback = getattr(spider, callback)
                if url_str.startswith('@'):
                    urls = selector(url_str[1:]).extract()
                else:
                    urls = [url_str]
                for url in urls:
                    yield Request(url, callback=f_callback)
        if 'follows' in meta:
            from scrapy.utils.response import get_base_url
            from urllib.parse import urljoin
            base_url = get_base_url(response)
            for url_str, callback in meta['follows'].items():
                f_callback = getattr(spider, callback)
                if url_str.startswith('@'):
                    urls = [urljoin(base_url, url) for url in selector(
                        url_str[1:]).extract()]
                else:
                    urls = [urljoin(base_url, url_str)]
                for url in urls:
                    yield Request(url, callback=f_callback)

<mask token>


class ParserCompose(ComposeField, ParserField):
    _fields = None

    def __init__(self, spider=None, **kwargs):
        super(ParserCompose, self).__init__(**kwargs)
        self.spider = spider
        self.composed = self.get_endpoints

    @property
    def fields(self):
        if not self._fields:
            self._fields = [ParserFields.fields[v](key=k, value=v) for k, v in
                self.value.items()]
        return self._fields

    @property
    def has_endpoints(self):
        return bool(self.meta)

    def get_context(self, response):
        ctx = {}
        for field in self.fields:
            ctx.update(field.get_context(response))
        return ctx

    def get_endpoints(self, *response):
        if not self.has_endpoints:
            yield None
            return
        from scrapy import Request
        response = response[-1]
        meta = self.meta
        spider = self.spider
        selector = self.get_selector(response)
        if 'item' in meta:
            from scrapy_compose.utils.load import resource as load_resource
            yield load_resource(meta['item'])(**self.get_context(response))
        if 'items' in meta:
            for query, callback in meta['items'].items():
                f_callback = getattr(spider, callback)
                if query.startswith('@'):
                    for block in selector(query[1:]):
                        yield next(f_callback(block))
        if 'requests' in meta:
            for url_str, callback in meta['requests'].items():
                f_callback = getattr(spider, callback)
                if url_str.startswith('@'):
                    urls = selector(url_str[1:]).extract()
                else:
                    urls = [url_str]
                for url in urls:
                    yield Request(url, callback=f_callback)
        if 'follows' in meta:
            from scrapy.utils.response import get_base_url
            from urllib.parse import urljoin
            base_url = get_base_url(response)
            for url_str, callback in meta['follows'].items():
                f_callback = getattr(spider, callback)
                if url_str.startswith('@'):
                    urls = [urljoin(base_url, url) for url in selector(
                        url_str[1:]).extract()]
                else:
                    urls = [urljoin(base_url, url_str)]
                for url in urls:
                    yield Request(url, callback=f_callback)

from .fields import ComposeField
from ..parser.fields import ParserFields, ParserField


class ParserCompose(ComposeField, ParserField):
    _fields = None

    def __init__(self, spider=None, **kwargs):
        super(ParserCompose, self).__init__(**kwargs)
        self.spider = spider
        self.composed = self.get_endpoints

    @property
    def fields(self):
        if not self._fields:
            self._fields = [ParserFields.fields[v](key=k, value=v) for k, v in
                self.value.items()]
        return self._fields

    @property
    def has_endpoints(self):
        return bool(self.meta)

    def get_context(self, response):
        ctx = {}
        for field in self.fields:
            ctx.update(field.get_context(response))
        return ctx

    def get_endpoints(self, *response):
        if not self.has_endpoints:
            yield None
            return
        from scrapy import Request
        response = response[-1]
        meta = self.meta
        spider = self.spider
        selector = self.get_selector(response)
        if 'item' in meta:
            from scrapy_compose.utils.load import resource as load_resource
            yield load_resource(meta['item'])(**self.get_context(response))
        if 'items' in meta:
            for query, callback in meta['items'].items():
                f_callback = getattr(spider, callback)
                if query.startswith('@'):
                    for block in selector(query[1:]):
                        yield next(f_callback(block))
        if 'requests' in meta:
            for url_str, callback in meta['requests'].items():
                f_callback = getattr(spider, callback)
                if url_str.startswith('@'):
                    urls = selector(url_str[1:]).extract()
                else:
                    urls = [url_str]
                for url in urls:
                    yield Request(url, callback=f_callback)
        if 'follows' in meta:
            from scrapy.utils.response import get_base_url
            from urllib.parse import urljoin
            base_url = get_base_url(response)
            for url_str, callback in meta['follows'].items():
                f_callback = getattr(spider, callback)
                if url_str.startswith('@'):
                    urls = [urljoin(base_url, url) for url in selector(
                        url_str[1:]).extract()]
                else:
                    urls = [urljoin(base_url, url_str)]
                for url in urls:
                    yield Request(url, callback=f_callback)


from .fields import ComposeField
from ..parser.fields import ParserFields, ParserField

class ParserCompose( ComposeField, ParserField ):

	_fields = None

	def __init__( self, spider = None, **kwargs ):
		super( ParserCompose, self ).__init__( **kwargs )

		self.spider = spider
		self.composed = self.get_endpoints

	@property
	def fields( self ):
		if not self._fields:
			self._fields = [
				ParserFields.fields[ v ]( key = k, value = v )
				for k, v in self.value.items()
			]
		return self._fields

	@property
	def has_endpoints( self ):
		return bool( self.meta )

	def get_context( self, response ):
		ctx = {}
		for field in self.fields:
			ctx.update( field.get_context( response ) )
		return ctx

	def get_endpoints( self, *response ):

		if not self.has_endpoints:
			yield None
			return

		from scrapy import Request

		response = response[-1]
		meta = self.meta
		spider = self.spider

		selector = self.get_selector( response )

		if "item" in meta:
			from scrapy_compose.utils.load import resource as load_resource
			yield load_resource( meta[ "item" ] )( **self.get_context( response ) )

		if "items" in meta:
			for query, callback in meta[ "items" ].items():
				f_callback = getattr( spider, callback )
				if query.startswith( "@" ):
					for block in selector( query[1:] ):
						yield next( f_callback( block ) )

		if "requests" in meta:
			for url_str, callback in meta[ "requests" ].items():
				f_callback = getattr( spider, callback )

				if url_str.startswith( "@" ):
					# realize will return string data directly if len( selected ) is 1
					urls = selector( url_str[1:] ).extract()
				else:
					urls = [ url_str ]

				for url in urls:
					yield Request( url, callback = f_callback )


		if "follows" in meta:
			from scrapy.utils.response import get_base_url
			from urllib.parse import urljoin

			base_url = get_base_url( response )
			for url_str, callback in meta[ "follows" ].items():
				f_callback = getattr( spider, callback )

				if url_str.startswith( "@" ):
					# realize will return string data directly if len( selected ) is 1
					urls = [ urljoin( base_url, url ) for url in selector( url_str[1:] ).extract() ]
				else:
					urls = [ urljoin( base_url, url_str ) ]

				for url in urls:
					yield Request( url, callback = f_callback )
