harry/coursehub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Seed the vendor catalogue into SQL and add an optional AI course advisor

Browse Source 
Course metadata: a scan now gives every course a course_meta row from the
downloader's sidecar JSON when the folder has one, else from catalog_courses
matched by the folder name the downloader would create (folderKeyFor, a port
of ine.py's sanitize_name). tools/build-catalog-seed.php turns the INE export
into seeds/ine_catalog.sql, an idempotent patch that creates its own table and
upserts by id, so it can be applied to the live server before or after the
code and re-run freely. Course pages show the blurb, difficulty, vendor and
tags; search matches tags and blurbs; Settings reports how each course got its
details. Against the live library 163 of 179 courses match.

Course advisor, off by default: api/advisor.php answers "which course should I
take to learn X / what comes before Y" with courses from the library, in
order, plus vendor courses not yet downloaded. includes/ai_client.php is a
small curl wrapper for any OpenAI-format chat endpoint (DeepSeek V4 Flash by
default, no composer); the prompt is a byte-stable catalogue of the library so
the provider's prefix cache covers it. Every course id the model returns is
validated server-side, a bad JSON reply gets one retry then a plain-text
fallback, and a per-user daily cap with token counts lives in
advisor_requests. With AI_ENABLED false the app makes no outbound requests
and the box does not appear; users can also hide it per browser.

Claude-Session: https://claude.ai/code/session_012enquL2RZdjy3D5AcKP5yx
This commit is contained in:
Harwinder Sekhon
2026-09-08 18:36:43 -04:00
parent 04314327db
commit 59fab56359
14 changed files with 1548 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
includes/functions.php
+319 -9
View File
@@ -162,6 +162,290 @@ function migrateLibrarySchema(PDO $db): void
$db->exec("ALTER TABLE `$table` MODIFY path VARCHAR(700) COLLATE utf8mb4_bin NOT NULL");
}
}
// Course metadata: the vendor catalogue (seeded by seeds/ine_catalog.sql,
// which carries the same CREATE TABLE IF NOT EXISTS, so either may come
// first) and what the scan resolves per course from it.
$db->exec(catalogCoursesDdl());
$db->exec(courseMetaDdl());
}
/* ---------------------------------------------------------------------------
* Course metadata
*
* Two sources, neither of them the folder tree:
* - the vendor's catalogue, seeded into `catalog_courses` from an export
* (tools/build-catalog-seed.php -> seeds/ine_catalog.sql), and
* - a sidecar JSON the downloader leaves in a course folder once every file
* of the course made it (`<slug>.json`, same shape as a catalogue entry).
* A scan resolves one `course_meta` row per course from them: the sidecar is
* exact, the catalogue is matched by the folder name the downloader would
* have created (folderKeyFor). Courses that came from somewhere else simply
* have no row and appear as they always did.
* ------------------------------------------------------------------------- */
function catalogCoursesDdl(): string
{
return 'CREATE TABLE IF NOT EXISTS catalog_courses (
id VARCHAR(64) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
slug VARCHAR(255) NOT NULL,
folder_key VARCHAR(255) NOT NULL,
vendor VARCHAR(100) NULL,
difficulty VARCHAR(32) NULL,
tags TEXT NULL,
short_description TEXT NULL,
description TEXT NULL,
learning_paths TEXT NULL,
categories TEXT NULL,
duration_seconds INT UNSIGNED NULL,
url VARCHAR(500) NULL,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_cc_folder_key (folder_key)
) ENGINE=InnoDB';
}
function courseMetaDdl(): string
{
return "CREATE TABLE IF NOT EXISTS course_meta (
folder_id INT UNSIGNED PRIMARY KEY,
source ENUM('sidecar', 'catalog') NOT NULL,
catalog_id VARCHAR(64) NULL,
title VARCHAR(255) NULL,
vendor VARCHAR(100) NULL,
difficulty VARCHAR(32) NULL,
tags TEXT NULL,
short_description TEXT NULL,
description TEXT NULL,
learning_paths TEXT NULL,
categories TEXT NULL,
duration_seconds INT UNSIGNED NULL,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT fk_cm_folder FOREIGN KEY (folder_id) REFERENCES folders(id) ON DELETE CASCADE
) ENGINE=InnoDB";
}
/**
* The folder name the downloader creates for a course title — a port of
* ine.py's sanitize_name(): runs of whitespace and hyphens become one
* underscore, anything that is not a word character or a dot is dropped,
* underscores are collapsed and trimmed. "Cisco SD-WAN Use Design, Cloud &
* Advanced Features" -> "Cisco_SD_WAN_Use_Design_Cloud_Advanced_Features".
* Applying it to a name that is already a key changes nothing, so a folder
* renamed by hand ("Border Gateway Protocol (BGP)") still lands on the same key.
*/
function folderKeyFor(string $name): string
{
$key = preg_replace('/[\s\-]+/u', '_', trim($name));
$key = preg_replace('/[^\w.]/u', '', $key);
$key = preg_replace('/_+/', '_', $key);
$key = trim($key, '_.');
return $key !== '' ? $key : 'unnamed';
}
/**
* The fields kept from a catalogue entry (or a sidecar, which has the same
* shape), whitelisted and capped. Everything else in the vendor's record —
* instructor bios, image URLs, access passes — is noise here.
*/
function catalogFields(array $c): array
{
$str = function ($v, int $max): ?string {
if (!is_string($v)) return null;
$v = trim(strip_tags($v));
return $v === '' ? null : mb_substr($v, 0, $max);
};
$names = function ($v, int $max): array {
$out = [];
foreach (is_array($v) ? $v : [] as $item) {
$n = is_array($item) ? ($item['name'] ?? null) : $item;
if (is_string($n) && trim($n) !== '') {
$out[] = mb_substr(trim($n), 0, 100);
}
if (count($out) >= $max) break;
}
return array_values(array_unique($out));
};
$vendor = $c['vendor'] ?? null;
$duration = $c['duration_seconds'] ?? $c['duration_in_seconds'] ?? null;
return [
'id' => $str($c['id'] ?? null, 64),
'name' => $str($c['name'] ?? null, 255),
'slug' => $str($c['slug'] ?? null, 255),
'vendor' => is_array($vendor) ? $str($vendor['name'] ?? null, 100) : $str($vendor, 100),
'difficulty' => $str($c['difficulty'] ?? null, 32),
'tags' => $names($c['tags'] ?? null, 40),
'short_description' => $str($c['short_description'] ?? null, 1000),
'description' => $str($c['description'] ?? null, 4000),
'learning_paths' => $names($c['learning_paths'] ?? null, 20),
'categories' => $names($c['categories'] ?? null, 20),
'duration_seconds' => is_numeric($duration) && $duration > 0 ? (int)$duration : null,
'url' => $str($c['url'] ?? null, 500),
];
}
/**
* The sidecar of a course folder, if one of its .json files is one: a course
* record has a name and a slug at the top level. The downloader also drops a
* .json per lab lesson deeper in the tree; those carry `lab_type` and are
* not what is wanted. Never fatal — a bad file is just no sidecar.
*/
function readCourseSidecar(string $absDir, array $jsonNames): ?array
{
foreach ($jsonNames as $entry) {
$file = $absDir . '/' . $entry;
if (!is_file($file) || (filesize($file) ?: 0) > 2 * 1024 * 1024) continue;
$data = json_decode((string)@file_get_contents($file), true);
if (!is_array($data) || isset($data['lab_type'])) continue;
if (!is_string($data['name'] ?? null) || !is_string($data['slug'] ?? null)) continue;
$fields = catalogFields($data);
if ($fields['name'] === null) continue;
return $fields;
}
return null;
}
/** A catalog_courses row in the catalogFields() shape (JSON columns decoded). */
function catalogRowFields(array $row): array
{
$list = fn($v) => is_string($v) ? (json_decode($v, true) ?: []) : [];
return [
'id' => $row['id'],
'name' => $row['name'],
'slug' => $row['slug'],
'vendor' => $row['vendor'],
'difficulty' => $row['difficulty'],
'tags' => $list($row['tags']),
'short_description' => $row['short_description'],
'description' => $row['description'],
'learning_paths' => $list($row['learning_paths']),
'categories' => $list($row['categories']),
'duration_seconds' => $row['duration_seconds'] !== null ? (int)$row['duration_seconds'] : null,
'url' => $row['url'],
];
}
/**
* Resolve `course_meta` for every course the scan just marked: the sidecar
* when the folder has one, else the catalogue entry whose folder key matches
* the folder name, else nothing. Runs inside the scan's transaction, after
* markCourseRoots(). `$jsonFiles` maps a folder's relative path to the .json
* names found in it. Returns the counts for the scan summary.
*/
function syncCourseMeta(PDO $db, string $root, array $jsonFiles): array
{
$stats = ['meta_sidecar' => 0, 'meta_catalog' => 0, 'meta_none' => 0];
$catalog = [];
foreach ($db->query('SELECT * FROM catalog_courses') as $row) {
$catalog[mb_strtolower($row['folder_key'])] = $row;
}
$upsert = $db->prepare(
'INSERT INTO course_meta (folder_id, source, catalog_id, title, vendor, difficulty, tags,
short_description, description, learning_paths, categories,
duration_seconds)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
source = VALUES(source), catalog_id = VALUES(catalog_id), title = VALUES(title),
vendor = VALUES(vendor), difficulty = VALUES(difficulty), tags = VALUES(tags),
short_description = VALUES(short_description), description = VALUES(description),
learning_paths = VALUES(learning_paths), categories = VALUES(categories),
duration_seconds = VALUES(duration_seconds)'
);
$seen = [];
$courses = $db->query('SELECT id, name, path FROM folders WHERE is_course = 1 AND is_missing = 0');
foreach ($courses as $f) {
$source = null;
$fields = readCourseSidecar($root . '/' . $f['path'], $jsonFiles[$f['path']] ?? []);
if ($fields !== null) {
$source = 'sidecar';
} else {
$key = mb_strtolower(folderKeyFor($f['name']));
if (isset($catalog[$key])) {
$fields = catalogRowFields($catalog[$key]);
$source = 'catalog';
}
}
if ($source === null) {
$stats['meta_none']++;
continue;
}
$json = fn(array $a) => json_encode($a, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
$upsert->execute([
(int)$f['id'], $source, $fields['id'], $fields['name'], $fields['vendor'],
$fields['difficulty'], $json($fields['tags']), $fields['short_description'],
$fields['description'], $json($fields['learning_paths']), $json($fields['categories']),
$fields['duration_seconds'],
]);
$seen[] = (int)$f['id'];
$stats['meta_' . $source]++;
}
if ($seen) {
$db->exec('DELETE FROM course_meta WHERE folder_id NOT IN (' . implode(',', $seen) . ')');
} else {
$db->exec('DELETE FROM course_meta');
}
return $stats;
}
/** How many catalogue courses have been seeded (0 until seeds/ine_catalog.sql is applied). */
function catalogCourseCount(): int
{
try {
return (int)db()->query('SELECT COUNT(*) FROM catalog_courses')->fetchColumn();
} catch (Throwable $e) {
return 0;
}
}
/** Every course_meta row, keyed by folder id. Empty until the first scan. */
function courseMetaAll(): array
{
try {
$rows = db()->query('SELECT * FROM course_meta')->fetchAll();
} catch (Throwable $e) {
return []; // table not created yet
}
$out = [];
foreach ($rows as $r) {
$out[(int)$r['folder_id']] = $r;
}
return $out;
}
function courseMeta(int $folderId): ?array
{
try {
$stmt = db()->prepare('SELECT * FROM course_meta WHERE folder_id = ?');
$stmt->execute([$folderId]);
return $stmt->fetch() ?: null;
} catch (Throwable $e) {
return null;
}
}
/** The shape a course_meta row takes in API responses. */
function courseMetaPublic(?array $row): ?array
{
if (!$row) return null;
$list = fn($v) => is_string($v) ? (json_decode($v, true) ?: []) : [];
return [
'source' => $row['source'],
'catalog_id' => $row['catalog_id'],
'title' => $row['title'],
'vendor' => $row['vendor'],
'difficulty' => $row['difficulty'],
'tags' => $list($row['tags']),
'short_description' => $row['short_description'],
'description' => $row['description'],
'learning_paths' => $list($row['learning_paths']),
'categories' => $list($row['categories']),
'duration_seconds' => $row['duration_seconds'] !== null ? (int)$row['duration_seconds'] : null,
];
}
/**
@@ -329,6 +613,7 @@ function scanLibraryLocked(PDO $db, string $root): array
$seenVideoIds = [];
$seenQuizIds = [];
$markers = []; // folder path => 'course' | 'category', from marker files
$jsonFiles = []; // folder path => [.json names in it], for course sidecars
$upsertFolder = $db->prepare(
'INSERT INTO folders (parent_id, name, path, depth, sort_order, is_missing)
@@ -363,7 +648,7 @@ function scanLibraryLocked(PDO $db, string $root): array
$walk = function (string $dir, ?int $parentId, string $relBase, int $depth)
use (&$walk, $db, $upsertFolder, $upsertVideo, $upsertQuiz,
&$stats, &$seenFolderIds, &$seenVideoIds, &$seenQuizIds, &$markers): void {
&$stats, &$seenFolderIds, &$seenVideoIds, &$seenQuizIds, &$markers, &$jsonFiles): void {
$entries = scandir($dir) ?: [];
// An empty ".course" / ".category" file overrides what markCourseRoots()
@@ -412,6 +697,11 @@ function scanLibraryLocked(PDO $db, string $root): array
$upsertQuiz->execute([$parentId, $name, $entry, $rel, $numQuestions, $quizSort++]);
$seenQuizIds[] = (int)$db->lastInsertId();
$stats['quizzes']++;
} elseif (is_file($full) && $relBase !== ''
&& strtolower(pathinfo($entry, PATHINFO_EXTENSION)) === 'json') {
// Remembered by folder; whether the folder is a course, and so
// whether this is its sidecar, is only known after the walk.
$jsonFiles[$relBase][] = $entry;
}
}
};
@@ -438,6 +728,7 @@ function scanLibraryLocked(PDO $db, string $root): array
// its contents are known, so it runs last — inside the same transaction,
// so no request ever sees a library with no courses in it.
markCourseRoots($db, $markers);
$stats += syncCourseMeta($db, $root, $jsonFiles);
$stats['courses'] = (int)$db->query(
'SELECT COUNT(*) FROM folders WHERE is_course = 1 AND is_missing = 0'
)->fetchColumn();
@@ -465,6 +756,9 @@ function scanLibraryLocked(PDO $db, string $root): array
}
appMetaSet('last_scan_at', gmdate('Y-m-d H:i:s'));
appMetaSet('course_meta_counts', json_encode([
'sidecar' => $stats['meta_sidecar'], 'catalog' => $stats['meta_catalog'], 'none' => $stats['meta_none'],
]));
return $stats;
}
@@ -1033,14 +1327,30 @@ function resetProgress(int $userId, string $scope, int $id = 0): array
function searchFolders(string $term, int $limit = 25): array
{
$like = likeContains($term);
$stmt = db()->prepare(
'SELECT * FROM folders
WHERE is_missing = 0
AND (name LIKE ? OR path COLLATE utf8mb4_unicode_ci LIKE ?)
ORDER BY depth, path
LIMIT ' . (int)$limit
);
$stmt->execute([$like, $like]);
// A course's catalogue tags and blurb count too, so "OMP" finds the SD-WAN
// course. An install that has never scanned has no course_meta table yet;
// it searches names alone until it does.
try {
$stmt = db()->prepare(
'SELECT f.* FROM folders f
LEFT JOIN course_meta m ON m.folder_id = f.id
WHERE f.is_missing = 0
AND (f.name LIKE ? OR f.path COLLATE utf8mb4_unicode_ci LIKE ?
OR m.tags LIKE ? OR m.short_description LIKE ?)
ORDER BY f.depth, f.path
LIMIT ' . (int)$limit
);
$stmt->execute([$like, $like, $like, $like]);
} catch (Throwable $e) {
$stmt = db()->prepare(
'SELECT * FROM folders
WHERE is_missing = 0
AND (name LIKE ? OR path COLLATE utf8mb4_unicode_ci LIKE ?)
ORDER BY depth, path
LIMIT ' . (int)$limit
);
$stmt->execute([$like, $like]);
}
return $stmt->fetchAll();
}