# /robots.txt file for http://webcrawler.com/ # # # Yahoo! User-agent: Slurp Crawl-delay: 20 # Naver User-Agent: Yetibot Crawl-Delay: 30 # vocus User-Agent: VocusBot Crawl-Delay: 30 # Nutch User-Agent: Nutch Crawl-Delay: 30 # JS-Kit User-Agent: JS-Kit Crawl-Delay: 30 User-agent: * Disallow: /kagawa_news/locality/article.aspx?id=20040814000110 Disallow: /kagawa_news/locality/print.aspx?id=20040814000110 Disallow: /kagawa_news/locality/article.aspx?id=20030814000087 Disallow: /kagawa_news/locality/print.aspx?id=20030814000087 Disallow: /national/social/article.aspx?id=20100204000143 Disallow: /national/social/print.aspx?id=20100204000143 Disallow: /national/social/article.aspx?id=20100204000273 Disallow: /national/social/print.aspx?id=20100204000273 Disallow: /national/social/article.aspx?id=20100205000036 Disallow: /national/social/print.aspx?id=20100205000036 Disallow: /national/social/article.aspx?id=20100223000412 Disallow: /national/social/print.aspx?id=20100223000412 Disallow: /national/social/article.aspx?id=20100224000262 Disallow: /national/social/print.aspx?id=20100224000262 Disallow: /kagawa_news/administration/article.aspx?id=20100309000103 Disallow: /kagawa_news/administration/print.aspx?id=20100309000103 Disallow: /national/social/article.aspx?id=20100304000224 Disallow: /national/social/print.aspx?id=20100304000224 Disallow: /national/social/article.aspx?id=20100531000168 Disallow: /national/social/print.aspx?id=20100531000168 Disallow: /national/social/article.aspx?id=20100603000187 Disallow: /kagawa_news/social/article.aspx?id=20100513000142 Disallow: /kagawa_news/social/print.aspx?id=20100513000142 Disallow: /kagawa_news/economy/article.aspx?id=20080408000089 Disallow: /kagawa_news/economy/print.aspx?id=20080408000089 Disallow: /national/social/article.aspx?id=20090703000447 Disallow: /national/social/print.aspx?id=20090703000447 Disallow: /national/social/article.aspx?id=20090623000250 Disallow: /national/social/print.aspx?id=20090623000250 Disallow: /national/social/article.aspx?id=20090831000262 Disallow: /national/social/print.aspx?id=20090831000262 Disallow: /national/social/article.aspx?id=20090226000200 Disallow: /national/social/print.aspx?id=20090226000200 Disallow: /national/social/print.aspx?id=20060826000300 Disallow: /national/social/article.aspx?id=20060826000300 Disallow: /national/social/article.aspx?id=20041216000465 Disallow: /national/social/print.aspx?id=20041216000465 Disallow: /national/social/article.aspx?id=20080731000488 Disallow: /national/social/print.aspx?id=20080731000488 Disallow: /national/social/article.aspx?id=20041109000263 Disallow: /national/social/article.aspx?id=20070319000153 Disallow: /national/social/print.aspx?id=20070319000153 Disallow: /national/social/article.aspx?id=20070319000333 Disallow: /national/social/print.aspx?id=20070319000333 Disallow: /national/social/article.aspx?id=20071107000156 Disallow: /national/social/print.aspx?id=20071107000156 Disallow: /national/social/article.aspx?id=20070206000004 Disallow: /national/social/print.aspx?id=20070206000004 Disallow: /national/social/article.aspx?id=20070206000206 Disallow: /national/social/print.aspx?id=20070206000206 Disallow: /national/social/article.aspx?id=20070226000298 Disallow: /national/social/print.aspx?id=20070226000298 Disallow: /national/social/article.aspx?id=20090224000336 Disallow: /national/social/print.aspx?id=20090224000336 Disallow: /national/social/article.aspx?id=20090224000071 Disallow: /national/social/print.aspx?id=20090224000071 Disallow: /kagawa_news/social/article.aspx?id=20090225000101 Disallow: /kagawa_news/social/print.aspx?id=20090225000101 Disallow: /kagawa_news/social/article.aspx?id=20090214000124 Disallow: /kagawa_news/social/article.aspx?id=20090214000124&ref=rss Disallow: /kagawa_news/social/print.aspx?id=20090214000124 Disallow: /news/news.aspx?id=20051124000420 Disallow: /national/social/article.aspx?id=20051124000420 Disallow: /national/social/print.aspx?id=20051124000420 Disallow: /national/social/article.aspx?id=20060612000222 Disallow: /national/social/print.aspx?id=20060612000222 Disallow: /news/news.asp?id=20011120000069 Disallow: /kagawa_news/economy/article.aspx?id=20011120000069 Disallow: /kagawa_news/economy/print.aspx?id=20011120000069 Disallow: /kagawa_news/economy/article.aspx?id=20070921000083 Disallow: /kagawa_news/economy/print.aspx?id=20070921000083 Disallow: /kagawa_news/social/print.aspx?id=20050824000151 Disallow: /kagawa_news/social/article.aspx?id=20050824000151 Disallow: /kagawa_news/social/print.aspx?id=20051108000114 Disallow: /kagawa_news/social/article.aspx?id=20051108000114 Disallow: /kagawa_news/administration/article.aspx?id=20081029000116&ref=rss Disallow: /kagawa_news/administration/article.aspx?id=20081029000116 Disallow: /kagawa_news/social/print.aspx?id=20081029000116 Disallow: /kagawa_news/social/article.aspx?id=20081029000116&ref=rss Disallow: /national/social/print.aspx?id=20081108000265 Disallow: /national/social/print.aspx?id=20070530000281 Disallow: /national/social/print.aspx?id=20071029000433 Disallow: /kagawa_news/locality/article.aspx?id=20080603000142 Disallow: /kagawa_news/locality/print.aspx?id=20080603000142 Disallow: /national/social/article.aspx?id=20090203000473 Disallow: /national/social/print.aspx?id=20090203000473 Disallow: /campaign/jifuri08/campaign.pdf