diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_concurrency.py b/docs/guides/code_examples/scrapy_migration/crawlee_concurrency.py new file mode 100644 index 0000000000..131a8fa2b1 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_concurrency.py @@ -0,0 +1,33 @@ +import asyncio + +from crawlee import ConcurrencySettings +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + # `ConcurrencySettings` replaces Scrapy's `CONCURRENT_REQUESTS` and + # `DOWNLOAD_DELAY`. + concurrency_settings = ConcurrencySettings( + # Start with this many parallel tasks. + desired_concurrency=5, + # Never run more than this many in parallel. + max_concurrency=20, + # Cap total throughput across the whole pool. + max_tasks_per_minute=120, + ) + + crawler = ParselCrawler( + concurrency_settings=concurrency_settings, + max_requests_per_crawl=50, + ) + + @crawler.router.default_handler + async def handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url}') + await context.enqueue_links(selector='li.next a') + + await crawler.run(['https://quotes.toscrape.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_crawlspider.py b/docs/guides/code_examples/scrapy_migration/crawlee_crawlspider.py new file mode 100644 index 0000000000..7e5e586342 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_crawlspider.py @@ -0,0 +1,42 @@ +import asyncio + +from crawlee import Glob +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler(max_requests_per_crawl=50) + + # The default handler plays the role of CrawlSpider's `rules`. It follows the + # pagination and enqueues each book detail page, routed by label. + @crawler.router.default_handler + async def listing_handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Listing {context.request.url}') + + # `selector` is the `restrict_css` analog. `include` is the `allow` analog: + # it keeps only URLs matching the given globs. + await context.enqueue_links( + selector='article.product_pod h3 a', + include=[Glob('https://books.toscrape.com/catalogue/**')], + label='book', + ) + + # Follow pagination without a label, like a `Rule` with no callback. + await context.enqueue_links(selector='li.next a') + + # Routed by the 'book' label, like a `Rule` with `callback='parse_book'`. + @crawler.router.handler('book') + async def book_handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Book {context.request.url}') + await context.push_data( + { + 'title': context.selector.css('h1::text').get(), + 'price': context.selector.css('p.price_color::text').get(), + } + ) + + await crawler.run(['https://books.toscrape.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_error_handling.py b/docs/guides/code_examples/scrapy_migration/crawlee_error_handling.py new file mode 100644 index 0000000000..e959ac2d52 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_error_handling.py @@ -0,0 +1,32 @@ +import asyncio + +from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler( + # Retry each failed request up to this many times (the `RetryMiddleware` analog). + max_request_retries=3, + max_requests_per_crawl=50, + ) + + @crawler.router.default_handler + async def handler(context: ParselCrawlingContext) -> None: + for quote in context.selector.css('div.quote'): + await context.push_data({'text': quote.css('span.text::text').get()}) + + # Runs between retries. It can inspect or adjust the request before the next try. + @crawler.error_handler + async def on_error(context: BasicCrawlingContext, error: Exception) -> None: + context.log.warning(f'Retrying {context.request.url}: {error}') + + # Runs once a request has exhausted all retries, like Scrapy's `errback`. + @crawler.failed_request_handler + async def on_failed(context: BasicCrawlingContext, error: Exception) -> None: + context.log.error(f'Giving up on {context.request.url}: {error}') + + await crawler.run(['https://quotes.toscrape.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_export.py b/docs/guides/code_examples/scrapy_migration/crawlee_export.py new file mode 100644 index 0000000000..1096b2a6b2 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_export.py @@ -0,0 +1,34 @@ +import asyncio + +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler(max_requests_per_crawl=50) + + @crawler.router.default_handler + async def handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url}') + + items = [ + { + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), + } + for quote in context.selector.css('div.quote') + ] + await context.push_data(items) + + await context.enqueue_links(selector='li.next a') + + await crawler.run(['https://quotes.toscrape.com/']) + + # Export the whole dataset to a file. The format follows the extension, + # which must be .json or .csv. It replaces Scrapy's `FEEDS` setting and + # the `-O output.json` CLI flag. + await crawler.export_data('quotes.json') + await crawler.export_data('quotes.csv') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_labels.py b/docs/guides/code_examples/scrapy_migration/crawlee_labels.py new file mode 100644 index 0000000000..264eba4172 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_labels.py @@ -0,0 +1,39 @@ +import asyncio + +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler(max_requests_per_crawl=50) + + # The default handler processes listing pages: the entry point and each + # paginated page. It routes author links to a separate handler by label. + @crawler.router.default_handler + async def listing_handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Listing {context.request.url}') + + # Enqueue author detail pages with a label. It replaces a Scrapy + # `Request(url, callback=self.parse_author)`. + await context.enqueue_links(selector='div.quote span a', label='author') + + # Follow the pagination link. + await context.enqueue_links(selector='li.next a') + + # This handler runs only for requests labeled 'author'. + @crawler.router.handler('author') + async def author_handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Author {context.request.url}') + + await context.push_data( + { + 'name': context.selector.css('h3.author-title::text').get(), + 'born': context.selector.css('span.author-born-date::text').get(), + 'bio': context.selector.css('div.author-description::text').get(), + } + ) + + await crawler.run(['https://quotes.toscrape.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_playwright.py b/docs/guides/code_examples/scrapy_migration/crawlee_playwright.py new file mode 100644 index 0000000000..0c3be6968d --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_playwright.py @@ -0,0 +1,33 @@ +import asyncio + +from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext + + +async def main() -> None: + # `PlaywrightCrawler` renders JavaScript in a real browser. It replaces the + # `scrapy-playwright` package, with browser support built into the framework. + crawler = PlaywrightCrawler( + headless=True, + max_requests_per_crawl=50, + ) + + @crawler.router.default_handler + async def handler(context: PlaywrightCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url}') + + # `context.page` is a Playwright `Page`. Query the rendered DOM directly. + for quote in await context.page.locator('div.quote').all(): + await context.push_data( + { + 'text': await quote.locator('span.text').text_content(), + 'author': await quote.locator('small.author').text_content(), + } + ) + + await context.enqueue_links(selector='li.next a') + + await crawler.run(['https://quotes.toscrape.com/js/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_post.py b/docs/guides/code_examples/scrapy_migration/crawlee_post.py new file mode 100644 index 0000000000..6b58567c79 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_post.py @@ -0,0 +1,48 @@ +import asyncio +from urllib.parse import urlencode + +from crawlee import Request +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler(max_requests_per_crawl=10) + + @crawler.router.default_handler + async def login_page(context: ParselCrawlingContext) -> None: + # The CSRF token is tied to the session cookie issued for this GET. + if not context.session: + raise RuntimeError('Session not found') + + token = context.selector.css('input[name="csrf_token"]::attr(value)').get() + form = {'csrf_token': token, 'username': 'user', 'password': 'pass'} + + # Crawlee's `payload` is the raw request body, so encode the fields yourself + # and set the `Content-Type`. Scrapy's `FormRequest` does both for you. + await context.add_requests( + [ + Request.from_url( + 'https://quotes.toscrape.com/login', + method='POST', + payload=urlencode(form), + headers={'content-type': 'application/x-www-form-urlencoded'}, + label='after-login', + # Bind the POST to the same session so its CSRF cookie matches. + session_id=context.session.id, + # The POST shares the GET's URL. Include the method and payload + # in the unique key, or the queue drops it as a duplicate. + use_extended_unique_key=True, + ) + ] + ) + + @crawler.router.handler('after-login') + async def after_login(context: ParselCrawlingContext) -> None: + logged_in = context.selector.css('a[href="/logout"]').get() is not None + await context.push_data({'logged_in': logged_in}) + + await crawler.run(['https://quotes.toscrape.com/login']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_proxy.py b/docs/guides/code_examples/scrapy_migration/crawlee_proxy.py new file mode 100644 index 0000000000..6d21d1fc1f --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_proxy.py @@ -0,0 +1,31 @@ +import asyncio + +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext +from crawlee.proxy_configuration import ProxyConfiguration + + +async def main() -> None: + # `ProxyConfiguration` replaces Scrapy's `HttpProxyMiddleware` and the + # `scrapy-rotating-proxies` package. The URLs rotate in a round-robin fashion. + proxy_configuration = ProxyConfiguration( + proxy_urls=[ + 'http://proxy-1.com/', + 'http://proxy-2.com/', + ] + ) + + crawler = ParselCrawler( + proxy_configuration=proxy_configuration, + max_requests_per_crawl=50, + ) + + @crawler.router.default_handler + async def handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url}') + await context.enqueue_links(selector='li.next a') + + await crawler.run(['https://quotes.toscrape.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_quotes.py b/docs/guides/code_examples/scrapy_migration/crawlee_quotes.py new file mode 100644 index 0000000000..fc3660f136 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_quotes.py @@ -0,0 +1,33 @@ +import asyncio + +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler(max_requests_per_crawl=50) + + # The default handler runs for the entry point and every paginated page. + @crawler.router.default_handler + async def handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url}') + + # `context.selector` is a Parsel `Selector`, the same object Scrapy exposes + # as `response`. CSS and XPath queries carry over unchanged. + items = [ + { + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), + 'tags': quote.css('div.tags a.tag::text').getall(), + } + for quote in context.selector.css('div.quote') + ] + await context.push_data(items) + + # Follow the pagination link to the next page. + await context.enqueue_links(selector='li.next a') + + await crawler.run(['https://quotes.toscrape.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_throttling.py b/docs/guides/code_examples/scrapy_migration/crawlee_throttling.py new file mode 100644 index 0000000000..62bba228a1 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_throttling.py @@ -0,0 +1,38 @@ +import asyncio + +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext +from crawlee.request_loaders import ThrottlingRequestManager +from crawlee.storages import RequestQueue + + +async def main() -> None: + # A regular request queue holds requests for non-throttled domains. + request_queue = await RequestQueue.open() + + # `ThrottlingRequestManager` wraps the queue and adds per-domain backoff. + # It reacts to HTTP 429 responses and `robots.txt` crawl-delay directives, which + # makes it the closest built-in analog to Scrapy's `AutoThrottle`. The crawler + # feeds it those signals, so you only list the domains to watch. + request_manager = ThrottlingRequestManager( + inner=request_queue, + domains=['quotes.toscrape.com'], + request_manager_opener=RequestQueue.open, + ) + + crawler = ParselCrawler( + request_manager=request_manager, + # Crawl-delay is only read when `robots.txt` handling is enabled. + respect_robots_txt_file=True, + max_requests_per_crawl=50, + ) + + @crawler.router.default_handler + async def handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url}') + await context.enqueue_links(selector='li.next a') + + await crawler.run(['https://quotes.toscrape.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_user_data.py b/docs/guides/code_examples/scrapy_migration/crawlee_user_data.py new file mode 100644 index 0000000000..faeb8a7133 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/crawlee_user_data.py @@ -0,0 +1,47 @@ +import asyncio +from urllib.parse import urljoin + +from crawlee import Request +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler(max_requests_per_crawl=50) + + @crawler.router.default_handler + async def listing_handler(context: ParselCrawlingContext) -> None: + requests: list[Request] = [] + + for book in context.selector.css('article.product_pod'): + href = book.css('h3 a::attr(href)').get() + if href is None: + continue + + # `user_data` travels with the request to the detail handler, + # the same role Scrapy's `cb_kwargs` plays. + requests.append( + Request.from_url( + urljoin(context.request.url, href), + label='book', + user_data={ + 'listing_price': book.css('p.price_color::text').get(), + }, + ) + ) + + await context.add_requests(requests) + + @crawler.router.handler('book') + async def book_handler(context: ParselCrawlingContext) -> None: + await context.push_data( + { + 'title': context.selector.css('h1::text').get(), + 'listing_price': context.request.user_data.get('listing_price'), + } + ) + + await crawler.run(['https://books.toscrape.com/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_authors.py b/docs/guides/code_examples/scrapy_migration/scrapy_authors.py new file mode 100644 index 0000000000..6ab3243590 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_authors.py @@ -0,0 +1,21 @@ +import scrapy + + +class AuthorsSpider(scrapy.Spider): + name = 'authors' + start_urls = ['https://quotes.toscrape.com/'] + + def parse(self, response): + for href in response.css('div.quote span a::attr(href)').getall(): + yield response.follow(href, callback=self.parse_author) + + next_page = response.css('li.next a::attr(href)').get() + if next_page is not None: + yield response.follow(next_page, callback=self.parse) + + def parse_author(self, response): + yield { + 'name': response.css('h3.author-title::text').get(), + 'born': response.css('span.author-born-date::text').get(), + 'bio': response.css('div.author-description::text').get(), + } diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py b/docs/guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py new file mode 100644 index 0000000000..fd9b9033de --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py @@ -0,0 +1,22 @@ +import scrapy + + +class BooksSpider(scrapy.Spider): + name = 'books' + start_urls = ['https://books.toscrape.com/'] + + def parse(self, response): + for book in response.css('article.product_pod'): + href = book.css('h3 a::attr(href)').get() + # Carry the listing price into the detail callback via `cb_kwargs`. + yield response.follow( + href, + callback=self.parse_book, + cb_kwargs={'listing_price': book.css('p.price_color::text').get()}, + ) + + def parse_book(self, response, listing_price): + yield { + 'title': response.css('h1::text').get(), + 'listing_price': listing_price, + } diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_concurrency.py b/docs/guides/code_examples/scrapy_migration/scrapy_concurrency.py new file mode 100644 index 0000000000..d724998450 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_concurrency.py @@ -0,0 +1,3 @@ +# settings.py +CONCURRENT_REQUESTS = 20 +DOWNLOAD_DELAY = 0.5 diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_crawlspider.py b/docs/guides/code_examples/scrapy_migration/scrapy_crawlspider.py new file mode 100644 index 0000000000..04abf2da60 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_crawlspider.py @@ -0,0 +1,23 @@ +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule + + +class BooksSpider(CrawlSpider): + name = 'books' + start_urls = ['https://books.toscrape.com/'] + + rules = ( + # Follow pagination, no callback. + Rule(LinkExtractor(restrict_css='li.next')), + # Extract each book detail page. + Rule( + LinkExtractor(restrict_css='article.product_pod h3', allow=r'/catalogue/'), + callback='parse_book', + ), + ) + + def parse_book(self, response): + yield { + 'title': response.css('h1::text').get(), + 'price': response.css('p.price_color::text').get(), + } diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_errback.py b/docs/guides/code_examples/scrapy_migration/scrapy_errback.py new file mode 100644 index 0000000000..e2a13cecdb --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_errback.py @@ -0,0 +1,18 @@ +import scrapy + + +class QuotesSpider(scrapy.Spider): + name = 'quotes' + start_urls = ['https://quotes.toscrape.com/'] + + def start_requests(self): + for url in self.start_urls: + # `RetryMiddleware` retries the request automatically before `errback` fires. + yield scrapy.Request(url, callback=self.parse, errback=self.on_error) + + def parse(self, response): + for quote in response.css('div.quote'): + yield {'text': quote.css('span.text::text').get()} + + def on_error(self, failure): + self.logger.error('Request failed: %s', failure.request.url) diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_export.py b/docs/guides/code_examples/scrapy_migration/scrapy_export.py new file mode 100644 index 0000000000..b7904a69b2 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_export.py @@ -0,0 +1,5 @@ +# settings.py +FEEDS = { + 'quotes.json': {'format': 'json'}, + 'quotes.csv': {'format': 'csv'}, +} diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_formrequest.py b/docs/guides/code_examples/scrapy_migration/scrapy_formrequest.py new file mode 100644 index 0000000000..98043fc0e7 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_formrequest.py @@ -0,0 +1,18 @@ +import scrapy + + +class LoginSpider(scrapy.Spider): + name = 'login' + start_urls = ['https://quotes.toscrape.com/login'] + + def parse(self, response): + # `from_response` picks up the hidden `csrf_token` field on its own, encodes + # the data as `form-urlencoded`, and sets the `Content-Type` header. + yield scrapy.FormRequest.from_response( + response, + formdata={'username': 'user', 'password': 'pass'}, + callback=self.after_login, + ) + + def after_login(self, response): + yield {'logged_in': bool(response.css('a[href="/logout"]'))} diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_playwright.py b/docs/guides/code_examples/scrapy_migration/scrapy_playwright.py new file mode 100644 index 0000000000..e7d3fca35a --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_playwright.py @@ -0,0 +1,27 @@ +import scrapy + + +class JsQuotesSpider(scrapy.Spider): + name = 'js-quotes' + + def start_requests(self): + yield scrapy.Request( + 'https://quotes.toscrape.com/js/', + meta={'playwright': True}, + ) + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), + } + + next_page = response.css('li.next a::attr(href)').get() + if next_page is not None: + # Every followed request has to opt back into rendering. + yield response.follow( + next_page, + callback=self.parse, + meta={'playwright': True}, + ) diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_playwright_settings.py b/docs/guides/code_examples/scrapy_migration/scrapy_playwright_settings.py new file mode 100644 index 0000000000..0bf4252861 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_playwright_settings.py @@ -0,0 +1,7 @@ +# Without these, `meta={'playwright': True}` is silently ignored and the page is +# fetched by the default downloader with no rendering. +DOWNLOAD_HANDLERS = { + 'http': 'scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler', + 'https': 'scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler', +} +TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_proxy.py b/docs/guides/code_examples/scrapy_migration/scrapy_proxy.py new file mode 100644 index 0000000000..53fd3e9bd6 --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_proxy.py @@ -0,0 +1,12 @@ +# settings.py +# Rotation needs the third-party `scrapy-rotating-proxies` package. The built-in +# `HttpProxyMiddleware` reads a single proxy from `request.meta` or the environment. +ROTATING_PROXY_LIST = [ + 'http://proxy-1.com/', + 'http://proxy-2.com/', +] + +DOWNLOADER_MIDDLEWARES = { + 'rotating_proxies.middlewares.RotatingProxyMiddleware': 610, + 'rotating_proxies.middlewares.BanDetectionMiddleware': 620, +} diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_quotes.py b/docs/guides/code_examples/scrapy_migration/scrapy_quotes.py new file mode 100644 index 0000000000..633c569f4c --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_quotes.py @@ -0,0 +1,18 @@ +import scrapy + + +class QuotesSpider(scrapy.Spider): + name = 'quotes' + start_urls = ['https://quotes.toscrape.com/'] + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), + 'tags': quote.css('div.tags a.tag::text').getall(), + } + + next_page = response.css('li.next a::attr(href)').get() + if next_page is not None: + yield response.follow(next_page, callback=self.parse) diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_throttling.py b/docs/guides/code_examples/scrapy_migration/scrapy_throttling.py new file mode 100644 index 0000000000..a0a57e2efa --- /dev/null +++ b/docs/guides/code_examples/scrapy_migration/scrapy_throttling.py @@ -0,0 +1,3 @@ +# settings.py +AUTOTHROTTLE_ENABLED = True +AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 diff --git a/docs/guides/scrapy_migration.mdx b/docs/guides/scrapy_migration.mdx new file mode 100644 index 0000000000..8bb0780c10 --- /dev/null +++ b/docs/guides/scrapy_migration.mdx @@ -0,0 +1,306 @@ +--- +id: scrapy-migration +title: Migrating from Scrapy +description: A guide for Scrapy users on how to move an existing project to Crawlee for Python, mapping Scrapy concepts to their Crawlee equivalents. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; +import CodeBlock from '@theme/CodeBlock'; + +import ScrapyQuotesSource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_quotes.py'; +import ScrapyAuthorsSource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_authors.py'; +import ScrapyCbKwargsSource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_cb_kwargs.py'; +import ScrapyCrawlSpiderSource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_crawlspider.py'; +import ScrapyExportSource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_export.py'; +import ScrapyConcurrencySource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_concurrency.py'; +import ScrapyThrottlingSource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_throttling.py'; +import ScrapyProxySource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_proxy.py'; +import ScrapyErrbackSource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_errback.py'; +import ScrapyFormRequestSource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_formrequest.py'; +import ScrapyPlaywrightSettingsSource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_playwright_settings.py'; +import ScrapyPlaywrightSource from '!!raw-loader!./code_examples/scrapy_migration/scrapy_playwright.py'; + +import QuotesExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_quotes.py'; +import LabelsExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_labels.py'; +import UserDataExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_user_data.py'; +import CrawlSpiderExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_crawlspider.py'; +import ExportExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_export.py'; +import ConcurrencyExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_concurrency.py'; +import ThrottlingExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_throttling.py'; +import ProxyExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_proxy.py'; +import ErrorHandlingExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_error_handling.py'; +import PostExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_post.py'; +import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/scrapy_migration/crawlee_playwright.py'; + +[Scrapy](https://scrapy.org/) and Crawlee solve the same problem, so most of what you know carries over. The concepts line up almost one to one, and the selector code often moves across without changes. Scrapy parses HTML with [Parsel](https://parsel.readthedocs.io/), and so does Crawlee's `ParselCrawler`. Your `response.css()` and `response.xpath()` queries keep working on `context.selector`. + +This guide maps Scrapy concepts to their Crawlee equivalents and rewrites a small spider step by step. The examples use `ParselCrawler` because it's the closest match for a Scrapy spider. The other HTTP option is `BeautifulSoupCrawler`, and the choice between them is covered in the [HTTP crawlers guide](./http-crawlers). For pages that need a browser, Crawlee has `PlaywrightCrawler`, covered in the [Playwright crawler guide](./playwright-crawler). + +Each section below pairs the Scrapy code with its Crawlee rewrite. Switch the tab to compare the two side by side. The Crawlee examples cap each run with `max_requests_per_crawl` to keep the demos short. The Scrapy equivalent is the `CLOSESPIDER_PAGECOUNT` setting from the built-in `CloseSpider` extension. + +## Installation + +Crawlee needs Python 3.10 or later. Install it with the Parsel extra: + +```bash +pip install 'crawlee[parsel]' +``` + +For the browser examples, also install Playwright: + +```bash +pip install 'crawlee[playwright]' +playwright install +``` + +## Conceptual differences + +Both frameworks give you a request scheduler, a duplicate filter, retries, and a way to pull data out of pages. The way you wire those pieces together differs. + +- Scrapy runs on the [Twisted](https://twisted.org/) reactor, and your callbacks are synchronous generators that `yield` items and requests. Crawlee runs on `asyncio`, and handlers are coroutines defined with `async def` that `await` helpers like `push_data` and `enqueue_links`. You write straight-line `async`/`await` code without a reactor. +- A Scrapy spider is a class with a `parse` method and callbacks. Crawlee routes requests through a `Router` that dispatches them to handler functions by their `label`. +- Browser rendering, proxy rotation, session management, and storage are part of the framework. In Scrapy you reach for `scrapy-playwright`, proxy middlewares, and feed exporters. In Crawlee these are built in and share one API. + +## Mapping key concepts + +| Scrapy | Crawlee | +| --- | --- | +| `scrapy.Spider` | `ParselCrawler` | +| `start_urls` / `start_requests()` | `crawler.run([...])` | +| `parse(self, response)` | `@crawler.router.default_handler` | +| `callback=self.parse_detail` | `@crawler.router.handler('detail')` | +| `cb_kwargs` / `Request.meta` | `Request.user_data` | +| `response.css()` / `response.xpath()` | `context.selector` | +| `yield {...}` | `push_data` | +| `scrapy.Item` | A plain `dict` | +| `response.follow()` / `yield Request(...)` | `enqueue_links` / `add_requests` | +| `dont_filter=True` | `Request.from_url(always_enqueue=True)` | +| `allowed_domains` | `enqueue_links(strategy=...)` | +| `scrapy.FormRequest` | `Request.from_url(method='POST', payload=...)` | +| Item pipelines | `Dataset` | +| Downloader / spider middlewares | `router.use()`, navigation hooks, HTTP clients | +| `settings.py` | `Configuration` + crawler arguments | +| `CONCURRENT_REQUESTS` / `DOWNLOAD_DELAY` | `ConcurrencySettings` | +| `AutoThrottle` | `ThrottlingRequestManager` | +| `DEPTH_LIMIT` | `max_crawl_depth` | +| `CLOSESPIDER_PAGECOUNT` | `max_requests_per_crawl` | +| `DOWNLOAD_TIMEOUT` | `navigation_timeout` | +| `ROBOTSTXT_OBEY` | `respect_robots_txt_file` | +| `RetryMiddleware` | `max_request_retries` | +| `errback` | `failed_request_handler` | +| `FEEDS` / feed exports | `export_data` | +| `CrawlSpider` / `Rules` | `enqueue_links(selector=..., label=...)` | +| `HttpProxyMiddleware` | `ProxyConfiguration` | +| `scrapy-playwright` | `PlaywrightCrawler` | +| `scrapy crawl myspider` | `python main.py` | + +## Migrating a spider + +Here's the canonical quotes spider from the [Scrapy tutorial](https://docs.scrapy.org/en/latest/intro/tutorial.html). It reads quotes from a listing page and follows the pagination link. The Crawlee version keeps the selectors untouched. The `parse` method becomes the default handler, `yield {...}` becomes `push_data`, and the manual pagination becomes a single `enqueue_links` call. + + + + + {ScrapyQuotesSource} + + + + + {QuotesExample} + + + + +Note that: + +- `context.selector` is the Parsel `Selector` that Scrapy calls `response`. Every CSS and XPath query works the same way. +- `enqueue_links` extracts the `href` from matching elements, resolves it against the current URL, and schedules it. It also skips URLs already seen, so you don't reimplement the duplicate filter. +- `push_data` writes to the default `Dataset`. There's no separate item pipeline to declare. + +## Routing with labels + +Scrapy routes pages by passing a `callback` to each request. A listing page hands its links to `parse_author`. In Crawlee you attach a `label` to the enqueued requests, then register a handler for that label. Each request goes to the matching handler, and unlabeled requests fall through to the default one. For details, see the [Request router guide](./request-router). + + + + + {ScrapyAuthorsSource} + + + + + {LabelsExample} + + + + +## Passing data between handlers + +A listing page often holds data you want to keep, then merge with fields from the detail page. Scrapy carries it through `cb_kwargs` (or `Request.meta`). Crawlee carries it through `Request.user_data`, which travels with the request and is available on `context.request.user_data` in the next handler. + +How you attach it depends on the data. When each link carries its own value, like a price that belongs to one book, create a request per URL with `Request.from_url(..., user_data=...)` and add them with `add_requests`. That's what the example below does. When a single value applies to the whole batch, skip the loop and pass `user_data` straight to `enqueue_links`. + + + + + {ScrapyCbKwargsSource} + + + + + {UserDataExample} + + + + +## Rule-based crawling + +Scrapy's `CrawlSpider` declares `Rules` with a `LinkExtractor` to follow links automatically and dispatch them to callbacks. Crawlee expresses the same intent with `enqueue_links`. The `restrict_css` and `restrict_xpaths` arguments map to `selector`, and the `allow` and `deny` patterns map to `include` and `exclude`, which accept `Glob` patterns or regular expressions. + + + + + {ScrapyCrawlSpiderSource} + + + + + {CrawlSpiderExample} + + + + +## Exporting data + +Scrapy writes results through the `FEEDS` setting or the `-O` command-line flag. Crawlee collects items in the default `Dataset` as you call `push_data`, then `export_data` writes the whole dataset once the run finishes. The format is chosen by the file extension, either `.json` or `.csv`. + + + + + {ScrapyExportSource} + + + + + {ExportExample} + + + + +The default (unnamed) `Dataset` stays on disk after the run finishes, but the next run clears it before starting, since `Configuration.purge_on_start` defaults to `True`. To keep data across runs, use a named dataset or turn that option off. For details, see the [Storages guide](./storages). To continue an interrupted crawl from where it stopped, see the [Resuming a paused crawl](../examples/resuming-paused-crawl) example. + +## Concurrency and throttling + +Scrapy tunes throughput through several settings. Crawlee gathers the fixed controls into `ConcurrencySettings` and scales concurrency automatically based on system resources, as described in the [Scaling crawlers guide](./scaling-crawlers). Use `max_tasks_per_minute` to cap the request rate. It's close to `DOWNLOAD_DELAY`, but it limits the whole pool rather than adding a fixed delay per domain. + + + + + {ScrapyConcurrencySource} + + + + + {ConcurrencyExample} + + + + +The closest built-in analog to the `AutoThrottle` extension is `ThrottlingRequestManager`, covered in the [Request throttling guide](./request-throttling). It differs in what drives the backoff. `AutoThrottle` adapts to measured latency, while the manager reacts to explicit signals: HTTP 429 replies and `robots.txt` crawl-delay directives. It wraps a `RequestQueue` and applies the backoff per domain. You list the domains to watch, and the crawler reports the signals to the manager on its own. + +Crawl-delay works only when the crawler fetches `robots.txt`. A project generated by `scrapy startproject` obeys `robots.txt`, but Crawlee doesn't by default. Pass `respect_robots_txt_file=True` to keep that behavior. The flag enforces the `Disallow` rules on its own. It reads crawl-delay only together with `ThrottlingRequestManager`, and without the manager the crawler logs a warning. + + + + + {ScrapyThrottlingSource} + + + + + {ThrottlingExample} + + + + +## Proxies + +Scrapy rotates proxies through a middleware or a third-party package. Crawlee takes a `ProxyConfiguration` and rotates the URLs for you. For tiered proxies, session-bound proxies, and integration with the Apify Proxy, see the [Proxy management guide](./proxy-management). + + + + + {ScrapyProxySource} + + + + + {ProxyExample} + + + + +## Error handling + +Scrapy retries failed requests with `RetryMiddleware` and reports terminal failures through `errback`. Crawlee retries with `max_request_retries`, runs an `error_handler` between attempts, and calls a `failed_request_handler` once the retries run out. For details, see the [Error handling guide](./error-handling). + + + + + {ScrapyErrbackSource} + + + + + {ErrorHandlingExample} + + + + +## Forms and login + +Scrapy submits forms with `FormRequest`, which encodes `formdata` as `form-urlencoded` and sets the header for you. Crawlee's `payload` takes the raw request body, so encode the fields yourself with `urllib.parse.urlencode` and set the `Content-Type` through `headers=`. For a full login flow with session reuse, see the [Logging in with a crawler guide](./logging-in-with-a-crawler). + + + + + {ScrapyFormRequestSource} + + + + + {PostExample} + + + + +## JavaScript rendering + +For pages that need a browser, Scrapy users add the `scrapy-playwright` package, which requires extra settings to register its download handlers and the asyncio reactor. Crawlee has browser support built in through `PlaywrightCrawler`, with nothing to register. The handler receives a [Playwright `Page`](https://playwright.dev/python/docs/api/class-page) on `context.page`, so you query the rendered DOM instead of a static response. For details, see the [Playwright crawler guide](./playwright-crawler). + + + + + {ScrapyPlaywrightSettingsSource} + + + {ScrapyPlaywrightSource} + + + + + {PlaywrightExample} + + + + +`PlaywrightCrawler` renders every request. In Scrapy you flag individual requests with `meta={'playwright': True}` and leave the rest on the plain downloader. For that mix in Crawlee, use `AdaptivePlaywrightCrawler`, which renders in a browser only when a page needs it. For details, see the [Adaptive Playwright crawler guide](./adaptive-playwright-crawler). + +## Conclusion + +Moving from Scrapy to Crawlee is mostly a mechanical rewrite. The spider class becomes a crawler with handlers, callbacks become labels, and `yield` becomes `push_data` or `enqueue_links`. The Parsel selectors carry over untouched. In return you get browser rendering, proxy rotation, and storage under one API, on top of `asyncio`. + +If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/pyproject.toml b/pyproject.toml index 3a250c8101..71747c1bc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,6 +230,10 @@ indent-style = "space" "**/docs/guides/code_examples/creating_web_archive/*.*" = [ "ASYNC230", # Ignore for simplicity of the example. ] +"**/docs/guides/code_examples/scrapy_migration/scrapy_*.py" = [ + "ANN", # Idiomatic Scrapy tutorial snippets are untyped; shown for comparison, not executed. + "RUF012", # Scrapy declares `start_urls` as a plain class-level list. +] [tool.ruff.lint.flake8-quotes] docstring-quotes = "double"