add special robots.txt middleware
adapted from
adds two new settings, both boolean:
* `DOCUMENTERS_AGGREGATOR_ROBOTSTXT_OBEY`: Obey robots.txt? (default: true)
* `DOCUMENTERS_AGGREGATOR_ROBOTSTXT_LOGONLY`: Log and continue? (default: true)
@@ -18,7 +18,9 @@ NEWSPIDER_MODULE = 'documenters_aggregator.spiders'
USER_AGENT = 'Documenters Aggregator (learn more and say hello at https://TKTK)'
# Obey robots.txt rules
-ROBOTSTXT_OBEY = True
+ROBOTSTXT_OBEY = False
+DOCUMENTERS_AGGREGATOR_ROBOTSTXT_OBEY = True
+DOCUMENTERS_AGGREGATOR_ROBOTSTXT_LOGONLY = True
# Disable cookies (enabled by default)
COOKIES_ENABLED = False
@@ -64,9 +66,9 @@ ITEM_PIPELINES = {
# Enable or disable downloader middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
-#DOWNLOADER_MIDDLEWARES = {
-# 'documenters_aggregator.middlewares.MyCustomDownloaderMiddleware': 543,
-#}
+DOWNLOADER_MIDDLEWARES = {
+ 'documenters_aggregator.middlewares.DocumentersAggregatorRobotsTxtMiddleware': 543,
+}
# Enable or disable extensions
# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
