4

スクリプトからスクレイピークローラーを実行するこのコードがあります( http://doc.scrapy.org/en/latest/topics/practices.html#run-scrapy-from-a-script )。しかし、うまくいきません。

from twisted.internet import reactor
from scrapy.crawler import Crawler
from scrapy import log,signals
from spiders.egov import EgovSpider
from scrapy.utils.project import get_project_settings

def run():
    spider =EgovSpider()
    settings = get_project_settings()
    crawler = Crawler(settings)
    crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
    crawler.configured
    crawler.crawl(spider)
    crawler.start()
    log.start()
    reactor.run()


from apscheduler.schedulers.twisted import TwistedScheduler
sched = TwistedScheduler()
sched.add_job(run, 'interval', seconds=10)
sched.start()

私のクモ:

import scrapy

class EgovSpider(scrapy.Spider):
    name = 'egov'
    start_urls = ['http://egov-buryatia.ru/index.php?id=1493']


    def parse(self, response):

        data = response.xpath("//div[@id='main_wrapper_content_news']//tr//text()").extract()
        print data
        print response.url
        f = open("vac.txt","a")
        for d in data:
            f.write(d.encode(encoding="UTF-8") + "\n")

        f.write(str(now))
        f.close()

行「reactor.run()」を置き換えると、スパイダーは 10 秒後に 1 回開始されます。

from twisted.internet import reactor
from scrapy.crawler import Crawler
from scrapy import log,signals
from spiders.egov import EgovSpider
from scrapy.utils.project import get_project_settings

def run():  
    spider =EgovSpider()
    settings = get_project_settings()
    crawler = Crawler(settings)
    crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
    crawler.configured
    crawler.crawl(spider)
    crawler.start()
    log.start()

from apscheduler.schedulers.twisted import TwistedScheduler
sched = TwistedScheduler()
sched.add_job(run, 'interval', seconds=10)
sched.start()
reactor.run()

私はpythonと英語の経験が少ないです:)助けてください。

4

1 に答える 1