File Manager Lite

Dir: /home/u540325668/domains/mccsociety.org/public_html/vendor/spatie/crawler/src

[..]

CrawlAllUrls.php (204 B)

CrawlInternalUrls.php (493 B)

CrawlObserver.php (1.3 KB)

CrawlQueue/

CrawlSubdomains.php (658 B)

CrawlUrl.php (858 B)

Crawler.php (12.28 KB)

CrawlerRobots.php (1.29 KB)

Exception/

Handlers/

LinkAdder.php (3.05 KB)

Edit: CrawlerRobots.php

<?php

namespace Spatie\Crawler;

use Psr\Http\Message\ResponseInterface;
use Spatie\Robots\RobotsHeaders;
use Spatie\Robots\RobotsMeta;

class CrawlerRobots
{
    /** @var \Spatie\Robots\RobotsHeaders */
    protected $robotsHeaders;

/** @var \Spatie\Robots\RobotsMeta */
    protected $robotsMeta;

/** @var bool */
    protected $mustRespectRobots;

public function __construct(ResponseInterface $response, bool $mustRespectRobots)
    {
        $this->robotsHeaders = RobotsHeaders::create($response->getHeaders());

$this->robotsMeta = RobotsMeta::create((string) $response->getBody());

$this->mustRespectRobots = $mustRespectRobots;
    }

public function mayIndex(): bool
    {
        if (! $this->mustRespectRobots) {
            return true;
        }

if (! $this->robotsHeaders->mayIndex()) {
            return false;
        }

if (! $this->robotsMeta->mayIndex()) {
            return false;
        }

return true;
    }

public function mayFollow(): bool
    {
        if (! $this->mustRespectRobots) {
            return true;
        }

if (! $this->robotsHeaders->mayFollow()) {
            return false;
        }

if (! $this->robotsMeta->mayFollow()) {
            return false;
        }

return true;
    }
}