Initial commit: RSS feed dashboard aggregating YouTube, Reddit and RT
- PHP 8 + PDO SQLite cache with on-demand TTL refresh and dedup by link hash - laminas-feed + Guzzle for RSS/Atom parsing and HTTP - Bootstrap 5 UI with source and time-range filters, AJAX load-more - User cron-free deployment under Apache/php-fpm; .htaccess hardening
This commit is contained in:
Harwinder Sekhon
@@ -0,0 +1 @@
|
||||
Require all denied
|
||||
@@ -0,0 +1,65 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App;
|
||||
|
||||
use PDO;
|
||||
|
||||
final class Database
|
||||
{
|
||||
public static function connect(string $path): PDO
|
||||
{
|
||||
$dir = dirname($path);
|
||||
if (!is_dir($dir) && !mkdir($dir, 0775, true) && !is_dir($dir)) {
|
||||
throw new \RuntimeException(sprintf('Unable to create database directory "%s".', $dir));
|
||||
}
|
||||
|
||||
$pdo = new PDO('sqlite:' . $path, null, null, [
|
||||
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
|
||||
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
|
||||
PDO::ATTR_EMULATE_PREPARES => false,
|
||||
]);
|
||||
|
||||
$pdo->exec('PRAGMA journal_mode = WAL');
|
||||
$pdo->exec('PRAGMA busy_timeout = 5000');
|
||||
$pdo->exec('PRAGMA foreign_keys = ON');
|
||||
|
||||
self::migrate($pdo);
|
||||
|
||||
return $pdo;
|
||||
}
|
||||
|
||||
private static function migrate(PDO $pdo): void
|
||||
{
|
||||
$pdo->exec(<<<'SQL'
|
||||
CREATE TABLE IF NOT EXISTS items (
|
||||
id TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
feed_label TEXT,
|
||||
feed_url TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
link TEXT NOT NULL,
|
||||
author TEXT,
|
||||
published_at TEXT,
|
||||
description TEXT,
|
||||
thumbnail TEXT,
|
||||
fetched_at TEXT NOT NULL
|
||||
)
|
||||
SQL);
|
||||
|
||||
$pdo->exec('CREATE INDEX IF NOT EXISTS idx_items_published ON items (published_at DESC)');
|
||||
$pdo->exec('CREATE INDEX IF NOT EXISTS idx_items_source ON items (source, published_at DESC)');
|
||||
|
||||
$pdo->exec(<<<'SQL'
|
||||
CREATE TABLE IF NOT EXISTS feeds (
|
||||
url TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
label TEXT,
|
||||
last_attempt_at TEXT,
|
||||
last_success_at TEXT,
|
||||
last_error TEXT
|
||||
)
|
||||
SQL);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App;
|
||||
|
||||
use DOMDocument;
|
||||
use DOMElement;
|
||||
use DOMXPath;
|
||||
use GuzzleHttp\Client;
|
||||
use Laminas\Feed\Reader\Reader;
|
||||
|
||||
final class FeedFetcher
|
||||
{
|
||||
private const MAX_DESCRIPTION_LENGTH = 600;
|
||||
private const MAX_ATTEMPTS = 2;
|
||||
|
||||
private Client $http;
|
||||
|
||||
/** @var array<string, float> */
|
||||
private array $lastRequestPerHost = [];
|
||||
|
||||
private int $hostIntervalMs;
|
||||
|
||||
public function __construct(private array $options = [])
|
||||
{
|
||||
$this->hostIntervalMs = (int) ($options['host_interval_ms'] ?? 1000);
|
||||
|
||||
$this->http = new Client([
|
||||
'timeout' => $options['request_timeout'] ?? 15,
|
||||
'connect_timeout' => $options['connect_timeout'] ?? 10,
|
||||
'http_errors' => false,
|
||||
'allow_redirects' => true,
|
||||
'headers' => [
|
||||
'User-Agent' => $options['user_agent'] ?? 'RSSFeed/1.0',
|
||||
'Accept' => 'application/atom+xml,application/rss+xml,application/xml,text/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language' => 'en-US,en;q=0.9',
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch and normalize a single feed.
|
||||
*
|
||||
* @return array{title: ?string, items: array<int, array<string, mixed>>}
|
||||
*/
|
||||
public function fetch(array $feed): array
|
||||
{
|
||||
$url = (string) $feed['url'];
|
||||
$source = (string) $feed['source'];
|
||||
$label = (string) ($feed['label'] ?? $url);
|
||||
|
||||
$xml = $this->download($url);
|
||||
$parsed = Reader::importString($xml);
|
||||
$extras = self::extractExtras($xml);
|
||||
|
||||
$now = gmdate('c');
|
||||
$items = [];
|
||||
|
||||
foreach ($parsed as $entry) {
|
||||
$link = self::safeUrl((string) $entry->getLink());
|
||||
if ($link === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$title = self::cleanText((string) $entry->getTitle());
|
||||
$description = (string) $entry->getDescription();
|
||||
if (trim($description) === '') {
|
||||
$description = (string) $entry->getContent();
|
||||
}
|
||||
|
||||
$thumbnail = $extras[$link]['thumbnail'] ?? null;
|
||||
$mediaDescription = $extras[$link]['description'] ?? null;
|
||||
if (trim($description) === '' && $mediaDescription !== null) {
|
||||
$description = $mediaDescription;
|
||||
}
|
||||
|
||||
if ($thumbnail === null) {
|
||||
$enclosure = $entry->getEnclosure();
|
||||
if ($enclosure !== null && self::isImageEnclosure($enclosure)) {
|
||||
$thumbnail = (string) $enclosure->url;
|
||||
}
|
||||
}
|
||||
|
||||
if ($source === 'youtube' && $thumbnail === null) {
|
||||
$videoId = self::youtubeVideoId($link);
|
||||
if ($videoId !== null) {
|
||||
$thumbnail = 'https://i.ytimg.com/vi/' . $videoId . '/mqdefault.jpg';
|
||||
}
|
||||
}
|
||||
|
||||
$date = $entry->getDateCreated() ?: $entry->getDateModified();
|
||||
|
||||
$items[] = [
|
||||
'id' => hash('sha256', $link),
|
||||
'source' => $source,
|
||||
'feed_label' => $label,
|
||||
'feed_url' => $url,
|
||||
'title' => $title !== '' ? $title : '(untitled)',
|
||||
'link' => $link,
|
||||
'author' => self::authorName($entry->getAuthor()),
|
||||
'published_at' => $date?->setTimezone(new \DateTimeZone('UTC'))->format('c'),
|
||||
'description' => self::cleanText($description, self::MAX_DESCRIPTION_LENGTH),
|
||||
'thumbnail' => self::safeUrl($thumbnail),
|
||||
'fetched_at' => $now,
|
||||
];
|
||||
}
|
||||
|
||||
return ['title' => (string) $parsed->getTitle(), 'items' => $items];
|
||||
}
|
||||
|
||||
private function download(string $url): string
|
||||
{
|
||||
$attempt = 0;
|
||||
$lastStatus = 0;
|
||||
|
||||
while ($attempt < self::MAX_ATTEMPTS) {
|
||||
$attempt++;
|
||||
$this->throttle($url);
|
||||
$response = $this->http->get($url);
|
||||
$lastStatus = $response->getStatusCode();
|
||||
|
||||
if ($lastStatus === 200) {
|
||||
return (string) $response->getBody();
|
||||
}
|
||||
|
||||
// Only retry transient server errors; client errors (incl. 429) are
|
||||
// handled by the caller with a staggered cooldown.
|
||||
if ($lastStatus < 500) {
|
||||
break;
|
||||
}
|
||||
|
||||
if ($attempt < self::MAX_ATTEMPTS) {
|
||||
$retryAfter = (int) $response->getHeaderLine('Retry-After');
|
||||
$delay = $retryAfter > 0 ? min($retryAfter, 3) : 1;
|
||||
sleep($delay);
|
||||
}
|
||||
}
|
||||
|
||||
throw new \RuntimeException(sprintf('HTTP %d while fetching %s', $lastStatus, $url));
|
||||
}
|
||||
|
||||
/**
|
||||
* Space out requests to the same host to avoid provider rate limiting.
|
||||
*/
|
||||
private function throttle(string $url): void
|
||||
{
|
||||
$host = (string) parse_url($url, PHP_URL_HOST);
|
||||
if ($host === '' || $this->hostIntervalMs <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
$now = microtime(true);
|
||||
if (isset($this->lastRequestPerHost[$host])) {
|
||||
$elapsedMs = ($now - $this->lastRequestPerHost[$host]) * 1000;
|
||||
$waitMs = $this->hostIntervalMs - $elapsedMs;
|
||||
if ($waitMs > 0) {
|
||||
usleep((int) ($waitMs * 1000));
|
||||
}
|
||||
}
|
||||
|
||||
$this->lastRequestPerHost[$host] = microtime(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract media thumbnails / descriptions from the raw XML, which the
|
||||
* laminas reader does not expose (YouTube media: RSS extension).
|
||||
*
|
||||
* @return array<string, array{thumbnail: ?string, description: ?string}>
|
||||
*/
|
||||
private static function extractExtras(string $xml): array
|
||||
{
|
||||
$dom = new DOMDocument();
|
||||
$previous = libxml_use_internal_errors(true);
|
||||
$loaded = $dom->loadXML($xml, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING);
|
||||
libxml_clear_errors();
|
||||
libxml_use_internal_errors($previous);
|
||||
|
||||
if (!$loaded) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$xpath = new DOMXPath($dom);
|
||||
$nodes = $xpath->query('//*[local-name()="entry" or local-name()="item"]');
|
||||
if ($nodes === false) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$map = [];
|
||||
foreach ($nodes as $node) {
|
||||
$link = self::nodeLink($xpath, $node);
|
||||
if ($link === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$thumbnail = null;
|
||||
$thumbNode = $xpath->query('.//*[local-name()="thumbnail"]', $node)->item(0);
|
||||
if ($thumbNode instanceof DOMElement) {
|
||||
$thumbnail = $thumbNode->getAttribute('url');
|
||||
}
|
||||
|
||||
$description = null;
|
||||
$descNode = $xpath->query('.//*[local-name()="group"]/*[local-name()="description"]', $node)->item(0);
|
||||
if ($descNode !== null) {
|
||||
$description = $descNode->textContent;
|
||||
}
|
||||
|
||||
$map[$link] = [
|
||||
'thumbnail' => self::safeUrl($thumbnail),
|
||||
'description' => $description,
|
||||
];
|
||||
}
|
||||
|
||||
return $map;
|
||||
}
|
||||
|
||||
private static function nodeLink(DOMXPath $xpath, \DOMNode $node): string
|
||||
{
|
||||
$links = $xpath->query('./*[local-name()="link"]', $node);
|
||||
if ($links === false) {
|
||||
return '';
|
||||
}
|
||||
|
||||
$fallback = '';
|
||||
foreach ($links as $link) {
|
||||
if ($link instanceof DOMElement && $link->hasAttribute('href')) {
|
||||
$href = trim($link->getAttribute('href'));
|
||||
if ($link->getAttribute('rel') === 'alternate') {
|
||||
return $href;
|
||||
}
|
||||
$fallback = $fallback ?: $href;
|
||||
} else {
|
||||
$fallback = $fallback ?: trim($link->textContent);
|
||||
}
|
||||
}
|
||||
|
||||
return $fallback;
|
||||
}
|
||||
|
||||
private static function authorName(array|string|null $author): ?string
|
||||
{
|
||||
$name = is_array($author) ? ($author['name'] ?? null) : $author;
|
||||
if (!is_string($name)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$name = ltrim(self::cleanText($name), '/');
|
||||
|
||||
return $name === '' ? null : $name;
|
||||
}
|
||||
|
||||
private static function youtubeVideoId(string $link): ?string
|
||||
{
|
||||
if (preg_match('~[?&]v=([A-Za-z0-9_-]{6,})~', $link, $m)) {
|
||||
return $m[1];
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static function cleanText(string $text, int $limit = 0): string
|
||||
{
|
||||
$text = html_entity_decode(strip_tags($text), ENT_QUOTES | ENT_HTML5, 'UTF-8');
|
||||
$text = preg_replace('/\s+/u', ' ', $text) ?? $text;
|
||||
$text = trim($text);
|
||||
|
||||
if ($limit > 0 && mb_strlen($text) > $limit) {
|
||||
$text = mb_substr($text, 0, $limit) . '…';
|
||||
}
|
||||
|
||||
return $text;
|
||||
}
|
||||
|
||||
private static function isImageEnclosure(object $enclosure): bool
|
||||
{
|
||||
$url = isset($enclosure->url) ? (string) $enclosure->url : '';
|
||||
$type = isset($enclosure->type) ? (string) $enclosure->type : '';
|
||||
|
||||
if ($url === '' || self::safeUrl($url) === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (str_starts_with($type, 'image/')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$path = strtolower((string) parse_url($url, PHP_URL_PATH));
|
||||
|
||||
return (bool) preg_match('/\.(jpe?g|png|gif|webp|avif)$/', $path);
|
||||
}
|
||||
|
||||
private static function safeUrl(?string $url): ?string
|
||||
{
|
||||
if ($url === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$url = trim($url);
|
||||
$scheme = strtolower((string) parse_url($url, PHP_URL_SCHEME));
|
||||
|
||||
return in_array($scheme, ['http', 'https'], true) ? $url : null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App;
|
||||
|
||||
use PDO;
|
||||
|
||||
final class FeedRepository
|
||||
{
|
||||
public function __construct(private PDO $pdo)
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep the feeds table in sync with the configured feed list.
|
||||
*
|
||||
* @param array<int, array<string, mixed>> $feeds
|
||||
*/
|
||||
public function syncFeeds(array $feeds): void
|
||||
{
|
||||
$stmt = $this->pdo->prepare(<<<'SQL'
|
||||
INSERT INTO feeds (url, source, label)
|
||||
VALUES (:url, :source, :label)
|
||||
ON CONFLICT(url) DO UPDATE SET source = excluded.source, label = excluded.label
|
||||
SQL);
|
||||
|
||||
foreach ($feeds as $feed) {
|
||||
$stmt->execute([
|
||||
':url' => (string) $feed['url'],
|
||||
':source' => (string) $feed['source'],
|
||||
':label' => (string) ($feed['label'] ?? $feed['url']),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the subset of feeds whose last attempt is missing or older than TTL.
|
||||
* Feeds that previously failed use a shorter retry interval.
|
||||
*
|
||||
* @param array<int, array<string, mixed>> $feeds
|
||||
* @return array<int, array<string, mixed>>
|
||||
*/
|
||||
public function staleFeeds(array $feeds, int $ttlSeconds, int $errorTtlSeconds): array
|
||||
{
|
||||
$attempts = [];
|
||||
foreach ($this->pdo->query('SELECT url, last_attempt_at, last_error FROM feeds') as $row) {
|
||||
$attempts[(string) $row['url']] = $row;
|
||||
}
|
||||
|
||||
$now = time();
|
||||
$stale = [];
|
||||
foreach ($feeds as $feed) {
|
||||
$row = $attempts[(string) $feed['url']] ?? null;
|
||||
$last = $row['last_attempt_at'] ?? null;
|
||||
|
||||
if ($last === null) {
|
||||
$stale[] = $feed;
|
||||
continue;
|
||||
}
|
||||
|
||||
$age = $now - (int) strtotime((string) $last);
|
||||
$isError = ($row['last_error'] ?? null) !== null;
|
||||
$threshold = $ttlSeconds;
|
||||
|
||||
if ($isError) {
|
||||
// Stagger retries so rate-limited feeds do not all retry at once.
|
||||
$threshold = $errorTtlSeconds + (crc32((string) $feed['url']) % max(1, $errorTtlSeconds));
|
||||
}
|
||||
|
||||
if ($age >= $threshold) {
|
||||
$stale[] = $feed;
|
||||
}
|
||||
}
|
||||
|
||||
return $stale;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert new items, ignoring ones already stored (deduplicated by link hash).
|
||||
*
|
||||
* @param array<int, array<string, mixed>> $items
|
||||
*/
|
||||
public function upsertItems(array $items): int
|
||||
{
|
||||
if ($items === []) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$stmt = $this->pdo->prepare(<<<'SQL'
|
||||
INSERT OR IGNORE INTO items
|
||||
(id, source, feed_label, feed_url, title, link, author, published_at, description, thumbnail, fetched_at)
|
||||
VALUES
|
||||
(:id, :source, :feed_label, :feed_url, :title, :link, :author, :published_at, :description, :thumbnail, :fetched_at)
|
||||
SQL);
|
||||
|
||||
$inserted = 0;
|
||||
$this->pdo->beginTransaction();
|
||||
try {
|
||||
foreach ($items as $item) {
|
||||
$stmt->execute([
|
||||
':id' => $item['id'],
|
||||
':source' => $item['source'],
|
||||
':feed_label' => $item['feed_label'],
|
||||
':feed_url' => $item['feed_url'],
|
||||
':title' => $item['title'],
|
||||
':link' => $item['link'],
|
||||
':author' => $item['author'],
|
||||
':published_at' => $item['published_at'],
|
||||
':description' => $item['description'],
|
||||
':thumbnail' => $item['thumbnail'],
|
||||
':fetched_at' => $item['fetched_at'],
|
||||
]);
|
||||
$inserted += $stmt->rowCount();
|
||||
}
|
||||
$this->pdo->commit();
|
||||
} catch (\Throwable $e) {
|
||||
$this->pdo->rollBack();
|
||||
throw $e;
|
||||
}
|
||||
|
||||
return $inserted;
|
||||
}
|
||||
|
||||
public function recordAttempt(string $url, bool $success, ?string $error = null): void
|
||||
{
|
||||
$now = gmdate('c');
|
||||
|
||||
if ($success) {
|
||||
$stmt = $this->pdo->prepare(
|
||||
'UPDATE feeds SET last_attempt_at = :now, last_success_at = :now2, last_error = NULL WHERE url = :url'
|
||||
);
|
||||
$stmt->execute([':now' => $now, ':now2' => $now, ':url' => $url]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$stmt = $this->pdo->prepare(
|
||||
'UPDATE feeds SET last_attempt_at = :now, last_error = :error WHERE url = :url'
|
||||
);
|
||||
$stmt->execute([':now' => $now, ':error' => $error, ':url' => $url]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the shared WHERE clause for item queries.
|
||||
*
|
||||
* @param array<string, string> $params
|
||||
*/
|
||||
private function itemFilter(?string $source, ?string $since, array &$params): string
|
||||
{
|
||||
$where = [];
|
||||
|
||||
if ($source !== null) {
|
||||
$where[] = 'source = :source';
|
||||
$params[':source'] = $source;
|
||||
}
|
||||
|
||||
if ($since !== null) {
|
||||
$where[] = 'COALESCE(published_at, fetched_at) >= :since';
|
||||
$params[':since'] = $since;
|
||||
}
|
||||
|
||||
return $where === [] ? '' : ' WHERE ' . implode(' AND ', $where);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, array<string, mixed>>
|
||||
*/
|
||||
public function getItems(int $limit, int $offset, ?string $source = null, ?string $since = null): array
|
||||
{
|
||||
$limit = max(1, $limit);
|
||||
$offset = max(0, $offset);
|
||||
|
||||
$params = [];
|
||||
$sql = 'SELECT * FROM items' . $this->itemFilter($source, $since, $params)
|
||||
. ' ORDER BY COALESCE(published_at, fetched_at) DESC, id LIMIT :limit OFFSET :offset';
|
||||
|
||||
$stmt = $this->pdo->prepare($sql);
|
||||
foreach ($params as $key => $value) {
|
||||
$stmt->bindValue($key, $value, PDO::PARAM_STR);
|
||||
}
|
||||
$stmt->bindValue(':limit', $limit, PDO::PARAM_INT);
|
||||
$stmt->bindValue(':offset', $offset, PDO::PARAM_INT);
|
||||
$stmt->execute();
|
||||
|
||||
return $stmt->fetchAll();
|
||||
}
|
||||
|
||||
public function countItems(?string $source = null, ?string $since = null): int
|
||||
{
|
||||
$params = [];
|
||||
$sql = 'SELECT COUNT(*) FROM items' . $this->itemFilter($source, $since, $params);
|
||||
|
||||
$stmt = $this->pdo->prepare($sql);
|
||||
$stmt->execute($params);
|
||||
|
||||
return (int) $stmt->fetchColumn();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, array{url: string, label: string, source: string, last_success_at: ?string, last_error: ?string}>
|
||||
*/
|
||||
public function feedsStatus(): array
|
||||
{
|
||||
return $this->pdo->query('SELECT url, label, source, last_success_at, last_error FROM feeds ORDER BY label')->fetchAll();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App;
|
||||
|
||||
final class FeedService
|
||||
{
|
||||
/**
|
||||
* @param array<int, array<string, mixed>> $feeds
|
||||
*/
|
||||
public function __construct(
|
||||
private FeedRepository $repository,
|
||||
private FeedFetcher $fetcher,
|
||||
private array $feeds,
|
||||
private int $ttlSeconds,
|
||||
private int $errorRetryTtlSeconds = 300,
|
||||
private array $throttledHosts = [],
|
||||
private int $throttledHostMax = 1,
|
||||
) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh every feed whose cached copy is stale. Failures are recorded and
|
||||
* skipped so previously cached items remain available.
|
||||
*
|
||||
* @return array{attempted: int, succeeded: int, failed: int, new_items: int, errors: array<int, string>}
|
||||
*/
|
||||
public function refreshStale(): array
|
||||
{
|
||||
$this->repository->syncFeeds($this->feeds);
|
||||
|
||||
$report = [
|
||||
'attempted' => 0,
|
||||
'succeeded' => 0,
|
||||
'failed' => 0,
|
||||
'new_items' => 0,
|
||||
'errors' => [],
|
||||
];
|
||||
|
||||
$hostCounts = [];
|
||||
|
||||
foreach ($this->repository->staleFeeds($this->feeds, $this->ttlSeconds, $this->errorRetryTtlSeconds) as $feed) {
|
||||
$url = (string) $feed['url'];
|
||||
$host = (string) parse_url($url, PHP_URL_HOST);
|
||||
|
||||
if (in_array($host, $this->throttledHosts, true)) {
|
||||
$seen = $hostCounts[$host] ?? 0;
|
||||
if ($seen >= $this->throttledHostMax) {
|
||||
// Leave it stale so a later request picks it up.
|
||||
continue;
|
||||
}
|
||||
$hostCounts[$host] = $seen + 1;
|
||||
}
|
||||
|
||||
$report['attempted']++;
|
||||
|
||||
try {
|
||||
$result = $this->fetcher->fetch($feed);
|
||||
$report['new_items'] += $this->repository->upsertItems($result['items']);
|
||||
$this->repository->recordAttempt($url, true);
|
||||
$report['succeeded']++;
|
||||
} catch (\Throwable $e) {
|
||||
$this->repository->recordAttempt($url, false, $e->getMessage());
|
||||
$report['failed']++;
|
||||
$report['errors'][] = ($feed['label'] ?? $url) . ': ' . $e->getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
return $report;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array{items: array<int, array<string, mixed>>, page: int, page_size: int, total: int, has_more: bool}
|
||||
*/
|
||||
public function page(?string $source, int $page, int $pageSize, ?string $since = null): array
|
||||
{
|
||||
$page = max(1, $page);
|
||||
$pageSize = max(1, $pageSize);
|
||||
$offset = ($page - 1) * $pageSize;
|
||||
|
||||
$items = $this->repository->getItems($pageSize, $offset, $source, $since);
|
||||
$total = $this->repository->countItems($source, $since);
|
||||
|
||||
return [
|
||||
'items' => $items,
|
||||
'page' => $page,
|
||||
'page_size' => $pageSize,
|
||||
'total' => $total,
|
||||
'has_more' => ($offset + count($items)) < $total,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, array<string, mixed>>
|
||||
*/
|
||||
public function feedsStatus(): array
|
||||
{
|
||||
return $this->repository->feedsStatus();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, string>
|
||||
*/
|
||||
public function sources(): array
|
||||
{
|
||||
$sources = [];
|
||||
foreach ($this->feeds as $feed) {
|
||||
$sources[(string) $feed['source']] = true;
|
||||
}
|
||||
|
||||
return array_keys($sources);
|
||||
}
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace App;
|
||||
|
||||
final class View
|
||||
{
|
||||
public static function e(?string $value): string
|
||||
{
|
||||
return htmlspecialchars((string) $value, ENT_QUOTES | ENT_SUBSTITUTE, 'UTF-8');
|
||||
}
|
||||
|
||||
public static function sourceLabel(string $source): string
|
||||
{
|
||||
return match ($source) {
|
||||
'youtube' => 'YouTube',
|
||||
'reddit' => 'Reddit',
|
||||
'rt' => 'RT',
|
||||
default => ucfirst($source),
|
||||
};
|
||||
}
|
||||
|
||||
public static function sourceBadge(string $source): string
|
||||
{
|
||||
return match ($source) {
|
||||
'youtube' => 'text-bg-danger',
|
||||
'reddit' => 'text-bg-warning',
|
||||
'rt' => 'text-bg-dark',
|
||||
default => 'text-bg-secondary',
|
||||
};
|
||||
}
|
||||
|
||||
public static function sourceColor(string $source): string
|
||||
{
|
||||
return match ($source) {
|
||||
'youtube' => '#ff1f4b',
|
||||
'reddit' => '#ff6a00',
|
||||
'rt' => '#00c389',
|
||||
default => '#6a3dff',
|
||||
};
|
||||
}
|
||||
|
||||
public static function rangeLabel(string $range): string
|
||||
{
|
||||
return match ($range) {
|
||||
'1h' => '1 hour',
|
||||
'2h' => '2 hours',
|
||||
'6h' => '6 hours',
|
||||
'24h' => '24 hours',
|
||||
'7d' => '7 days',
|
||||
default => 'All time',
|
||||
};
|
||||
}
|
||||
|
||||
public static function render(string $template, array $data = []): string
|
||||
{
|
||||
$path = dirname(__DIR__) . '/templates/' . $template . '.php';
|
||||
if (!is_file($path)) {
|
||||
throw new \RuntimeException(sprintf('Template "%s" not found.', $template));
|
||||
}
|
||||
|
||||
extract($data, EXTR_SKIP);
|
||||
ob_start();
|
||||
include $path;
|
||||
|
||||
return (string) ob_get_clean();
|
||||
}
|
||||
|
||||
public static function timeAgo(?string $iso): string
|
||||
{
|
||||
if ($iso === null || $iso === '') {
|
||||
return '';
|
||||
}
|
||||
|
||||
$timestamp = strtotime($iso);
|
||||
if ($timestamp === false) {
|
||||
return '';
|
||||
}
|
||||
|
||||
$diff = time() - $timestamp;
|
||||
if ($diff < 60) {
|
||||
return 'just now';
|
||||
}
|
||||
|
||||
$units = [
|
||||
31536000 => 'y',
|
||||
2592000 => 'mo',
|
||||
604800 => 'w',
|
||||
86400 => 'd',
|
||||
3600 => 'h',
|
||||
60 => 'm',
|
||||
];
|
||||
|
||||
foreach ($units as $seconds => $label) {
|
||||
if ($diff >= $seconds) {
|
||||
return floor($diff / $seconds) . $label . ' ago';
|
||||
}
|
||||
}
|
||||
|
||||
return 'just now';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
require dirname(__DIR__) . '/vendor/autoload.php';
|
||||
|
||||
$config = require dirname(__DIR__) . '/config/app.php';
|
||||
$feeds = require dirname(__DIR__) . '/config/feeds.php';
|
||||
|
||||
$pdo = App\Database::connect($config['db_path']);
|
||||
$repository = new App\FeedRepository($pdo);
|
||||
$fetcher = new App\FeedFetcher($config);
|
||||
$service = new App\FeedService(
|
||||
$repository,
|
||||
$fetcher,
|
||||
$feeds,
|
||||
(int) $config['cache_ttl'],
|
||||
(int) ($config['error_retry_ttl'] ?? 300),
|
||||
(array) ($config['throttled_hosts'] ?? []),
|
||||
(int) ($config['throttled_host_max_per_refresh'] ?? 1)
|
||||
);
|
||||
|
||||
return [
|
||||
'config' => $config,
|
||||
'feeds' => $feeds,
|
||||
'pdo' => $pdo,
|
||||
'repository' => $repository,
|
||||
'fetcher' => $fetcher,
|
||||
'service' => $service,
|
||||
];
|
||||
Reference in New Issue
Block a user