- Improved the YouTube importer in [admin/youtube_import.php (line 154)](/Users/tyemeclifford/Documents/GH/mediaplayer/admin/youtube_import.php:154).
It now handles more YouTube layouts: Classic videoRenderer, gridVideoRenderer, and compactVideoRenderer Shorts-style reelItemRenderer Newer lockupViewModel channel grid objects Raw HTML fallback by extracting watch?v=, /shorts/, /embed/, and "videoId" matches A second scrape attempt using YouTube’s older grid query: view=0&sort=dd&flow=grid So instead of stopping at “no videos found” when YouTube changes the JSON shape, it now tries multiple extraction strategies and can still import proper YouTube embed URLs with fallback thumbnails.
This commit is contained in:
Ty Clifford
+220
-33
@@ -155,6 +155,10 @@ function yt_text($value): string {
|
||||
if (is_string($value)) return trim($value);
|
||||
if (!is_array($value)) return '';
|
||||
if (isset($value['simpleText'])) return trim((string)$value['simpleText']);
|
||||
if (isset($value['content'])) return trim((string)$value['content']);
|
||||
if (isset($value['text'])) return trim((string)$value['text']);
|
||||
if (isset($value['label'])) return trim((string)$value['label']);
|
||||
if (isset($value['accessibilityLabel'])) return trim((string)$value['accessibilityLabel']);
|
||||
if (isset($value['runs']) && is_array($value['runs'])) {
|
||||
$parts = [];
|
||||
foreach ($value['runs'] as $run) {
|
||||
@@ -168,6 +172,12 @@ function yt_text($value): string {
|
||||
return '';
|
||||
}
|
||||
|
||||
function yt_video_id(string $value): string {
|
||||
$value = trim($value);
|
||||
if (preg_match('/^[A-Za-z0-9_-]{11}$/', $value)) return $value;
|
||||
return '';
|
||||
}
|
||||
|
||||
function yt_best_thumbnail(array $renderer): string {
|
||||
$thumbs = $renderer['thumbnail']['thumbnails'] ?? [];
|
||||
if (!is_array($thumbs) || empty($thumbs)) return '';
|
||||
@@ -175,6 +185,36 @@ function yt_best_thumbnail(array $renderer): string {
|
||||
return isset($thumbs[0]['url']) ? html_entity_decode((string)$thumbs[0]['url'], ENT_QUOTES | ENT_HTML5, 'UTF-8') : '';
|
||||
}
|
||||
|
||||
function yt_collect_thumbnail_candidates($node, array &$candidates): void {
|
||||
if (!is_array($node)) return;
|
||||
|
||||
if (isset($node['url']) && is_string($node['url'])) {
|
||||
$url = html_entity_decode($node['url'], ENT_QUOTES | ENT_HTML5, 'UTF-8');
|
||||
if (preg_match('#https?://[^"\']*(?:ytimg|ggpht)[^"\']*#i', $url)) {
|
||||
$candidates[] = [
|
||||
'url' => $url,
|
||||
'width' => (int)($node['width'] ?? 0),
|
||||
'score' => str_contains($url, 'i.ytimg.com') ? 2 : 1,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($node as $value) {
|
||||
if (is_array($value)) yt_collect_thumbnail_candidates($value, $candidates);
|
||||
}
|
||||
}
|
||||
|
||||
function yt_best_thumbnail_any(array $node, string $video_id = ''): string {
|
||||
$candidates = [];
|
||||
yt_collect_thumbnail_candidates($node, $candidates);
|
||||
if ($candidates) {
|
||||
usort($candidates, fn($a, $b) => (($b['score'] <=> $a['score']) ?: ($b['width'] <=> $a['width'])));
|
||||
return $candidates[0]['url'];
|
||||
}
|
||||
|
||||
return $video_id !== '' ? 'https://i.ytimg.com/vi/' . rawurlencode($video_id) . '/hqdefault.jpg' : '';
|
||||
}
|
||||
|
||||
function yt_duration_seconds(string $duration): int {
|
||||
$duration = trim($duration);
|
||||
if ($duration === '' || stripos($duration, 'live') !== false) return 0;
|
||||
@@ -202,17 +242,113 @@ function yt_renderer_duration(array $renderer): string {
|
||||
return '';
|
||||
}
|
||||
|
||||
function yt_collect_renderers($node, array &$renderers): void {
|
||||
if (!is_array($node)) return;
|
||||
function yt_video_record(string $id, string $title, string $description = '', string $duration_text = '', string $thumbnail_url = ''): ?array {
|
||||
$id = yt_video_id($id);
|
||||
$title = trim($title);
|
||||
if ($id === '') return null;
|
||||
if ($title === '') $title = 'YouTube Video ' . $id;
|
||||
|
||||
foreach (['videoRenderer', 'gridVideoRenderer'] as $key) {
|
||||
return [
|
||||
'id' => $id,
|
||||
'title' => $title,
|
||||
'description' => trim($description),
|
||||
'duration' => yt_duration_seconds($duration_text),
|
||||
'duration_text' => trim($duration_text),
|
||||
'thumbnail_url' => $thumbnail_url ?: 'https://i.ytimg.com/vi/' . rawurlencode($id) . '/hqdefault.jpg',
|
||||
'watch_url' => 'https://www.youtube.com/watch?v=' . rawurlencode($id),
|
||||
];
|
||||
}
|
||||
|
||||
function yt_string_at_path(array $node, array $path): string {
|
||||
$current = $node;
|
||||
foreach ($path as $key) {
|
||||
if (!is_array($current) || !array_key_exists($key, $current)) return '';
|
||||
$current = $current[$key];
|
||||
}
|
||||
return yt_text($current);
|
||||
}
|
||||
|
||||
function yt_find_first_text_by_key($node, array $keys): string {
|
||||
if (!is_array($node)) return '';
|
||||
foreach ($keys as $key) {
|
||||
if (isset($node[$key])) {
|
||||
$text = yt_text($node[$key]);
|
||||
if ($text !== '') return $text;
|
||||
}
|
||||
}
|
||||
foreach ($node as $value) {
|
||||
if (is_array($value)) {
|
||||
$text = yt_find_first_text_by_key($value, $keys);
|
||||
if ($text !== '') return $text;
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function yt_video_from_reel_renderer(array $renderer): ?array {
|
||||
$id = yt_video_id((string)($renderer['videoId'] ?? ''));
|
||||
if ($id === '') return null;
|
||||
$title = yt_text($renderer['headline'] ?? []);
|
||||
if ($title === '') $title = yt_text($renderer['accessibility']['accessibilityData']['label'] ?? []);
|
||||
if ($title === '') $title = yt_find_first_text_by_key($renderer, ['title', 'headline']);
|
||||
$views = yt_text($renderer['viewCountText'] ?? []);
|
||||
$description = $views !== '' ? $views : '';
|
||||
return yt_video_record($id, $title, $description, '', yt_best_thumbnail_any($renderer, $id));
|
||||
}
|
||||
|
||||
function yt_video_from_lockup_model(array $model): ?array {
|
||||
$id = yt_video_id((string)($model['contentId'] ?? $model['videoId'] ?? ''));
|
||||
if ($id === '') {
|
||||
$url = yt_find_first_text_by_key($model, ['url', 'webCommandMetadata']);
|
||||
if (preg_match('/(?:watch\?v=|shorts\/)([A-Za-z0-9_-]{11})/', $url, $m)) $id = $m[1];
|
||||
}
|
||||
if ($id === '') return null;
|
||||
|
||||
$title = yt_string_at_path($model, ['metadata', 'lockupMetadataViewModel', 'title']);
|
||||
if ($title === '') $title = yt_find_first_text_by_key($model, ['title', 'headline', 'contentTitle']);
|
||||
|
||||
$metadata = [];
|
||||
foreach (['metadata', 'subtitle', 'secondarySubtitle', 'viewCountText', 'publishedTimeText'] as $key) {
|
||||
$text = yt_find_first_text_by_key($model[$key] ?? [], ['content', 'simpleText', 'text']);
|
||||
if ($text !== '') $metadata[] = $text;
|
||||
}
|
||||
|
||||
$duration = yt_find_first_text_by_key($model, ['duration', 'lengthText', 'thumbnailOverlayTimeStatusRenderer']);
|
||||
if ($duration !== '' && !preg_match('/\d/', $duration)) $duration = '';
|
||||
|
||||
return yt_video_record($id, $title, implode("\n", array_unique($metadata)), $duration, yt_best_thumbnail_any($model, $id));
|
||||
}
|
||||
|
||||
function yt_add_video(array $video, array &$videos, array &$seen, int $limit): void {
|
||||
if (isset($seen[$video['id']]) || count($videos) >= $limit) return;
|
||||
$seen[$video['id']] = true;
|
||||
$videos[] = $video;
|
||||
}
|
||||
|
||||
function yt_collect_video_items($node, array &$videos, array &$seen, int $limit): void {
|
||||
if (!is_array($node)) return;
|
||||
if (count($videos) >= $limit) return;
|
||||
|
||||
foreach (['videoRenderer', 'gridVideoRenderer', 'compactVideoRenderer'] as $key) {
|
||||
if (isset($node[$key]) && is_array($node[$key]) && !empty($node[$key]['videoId'])) {
|
||||
$renderers[] = $node[$key];
|
||||
$video = yt_video_from_renderer($node[$key]);
|
||||
if ($video) yt_add_video($video, $videos, $seen, $limit);
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($node['reelItemRenderer']) && is_array($node['reelItemRenderer'])) {
|
||||
$video = yt_video_from_reel_renderer($node['reelItemRenderer']);
|
||||
if ($video) yt_add_video($video, $videos, $seen, $limit);
|
||||
}
|
||||
|
||||
if (isset($node['lockupViewModel']) && is_array($node['lockupViewModel'])) {
|
||||
$video = yt_video_from_lockup_model($node['lockupViewModel']);
|
||||
if ($video) yt_add_video($video, $videos, $seen, $limit);
|
||||
}
|
||||
|
||||
foreach ($node as $value) {
|
||||
if (is_array($value)) yt_collect_renderers($value, $renderers);
|
||||
if (is_array($value)) yt_collect_video_items($value, $videos, $seen, $limit);
|
||||
if (count($videos) >= $limit) return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,15 +365,58 @@ function yt_video_from_renderer(array $renderer): ?array {
|
||||
$views = yt_text($renderer['viewCountText'] ?? []);
|
||||
$description_parts = array_filter([$snippet, trim($published . ($published && $views ? ' - ' : '') . $views)]);
|
||||
|
||||
return [
|
||||
'id' => $id,
|
||||
'title' => $title,
|
||||
'description' => implode("\n", $description_parts),
|
||||
'duration' => yt_duration_seconds($duration_text),
|
||||
'duration_text' => $duration_text,
|
||||
'thumbnail_url' => yt_best_thumbnail($renderer),
|
||||
'watch_url' => 'https://www.youtube.com/watch?v=' . rawurlencode($id),
|
||||
return yt_video_record($id, $title, implode("\n", $description_parts), $duration_text, yt_best_thumbnail($renderer));
|
||||
}
|
||||
|
||||
function yt_title_near_video_id(string $html, string $id): string {
|
||||
$quoted_id = preg_quote($id, '/');
|
||||
$patterns = [
|
||||
'/"videoId":"' . $quoted_id . '".{0,5000}?"title":\{"runs":\[\{"text":"([^"]+)"/s',
|
||||
'/"videoId":"' . $quoted_id . '".{0,5000}?"title":\{"simpleText":"([^"]+)"/s',
|
||||
'/"title":\{"runs":\[\{"text":"([^"]+)"\}\]\}.{0,5000}?"videoId":"' . $quoted_id . '"/s',
|
||||
'/"title":\{"simpleText":"([^"]+)"\}.{0,5000}?"videoId":"' . $quoted_id . '"/s',
|
||||
];
|
||||
foreach ($patterns as $pattern) {
|
||||
if (preg_match($pattern, $html, $m)) {
|
||||
$decoded = json_decode('"' . str_replace('"', '\"', $m[1]) . '"');
|
||||
return is_string($decoded) ? $decoded : html_entity_decode($m[1], ENT_QUOTES | ENT_HTML5, 'UTF-8');
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function yt_duration_near_video_id(string $html, string $id): string {
|
||||
$quoted_id = preg_quote($id, '/');
|
||||
$patterns = [
|
||||
'/"videoId":"' . $quoted_id . '".{0,5000}?"lengthText":\{"[^}]*"simpleText":"([^"]+)"/s',
|
||||
'/"lengthText":\{"[^}]*"simpleText":"([^"]+)".{0,5000}?"videoId":"' . $quoted_id . '"/s',
|
||||
];
|
||||
foreach ($patterns as $pattern) {
|
||||
if (preg_match($pattern, $html, $m)) return html_entity_decode($m[1], ENT_QUOTES | ENT_HTML5, 'UTF-8');
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function yt_videos_from_html(string $html, int $limit): array {
|
||||
preg_match_all('/(?:watch\?v=|shorts\/|embed\/|\"videoId\":\")([A-Za-z0-9_-]{11})/', $html, $matches);
|
||||
$ids = array_values(array_unique($matches[1] ?? []));
|
||||
$videos = [];
|
||||
$seen = [];
|
||||
foreach ($ids as $id) {
|
||||
if (count($videos) >= $limit) break;
|
||||
$title = yt_title_near_video_id($html, $id);
|
||||
$duration = yt_duration_near_video_id($html, $id);
|
||||
$video = yt_video_record($id, $title, '', $duration, 'https://i.ytimg.com/vi/' . rawurlencode($id) . '/hqdefault.jpg');
|
||||
if ($video) yt_add_video($video, $videos, $seen, $limit);
|
||||
}
|
||||
return $videos;
|
||||
}
|
||||
|
||||
function yt_channel_candidate_urls(string $url): array {
|
||||
return array_values(array_unique([
|
||||
$url,
|
||||
media_append_query($url, ['view' => '0', 'sort' => 'dd', 'flow' => 'grid']),
|
||||
]));
|
||||
}
|
||||
|
||||
function yt_scrape_channel_videos(string $input, int $limit): array {
|
||||
@@ -246,31 +425,39 @@ function yt_scrape_channel_videos(string $input, int $limit): array {
|
||||
return ['url' => '', 'videos' => [], 'error' => 'Enter a YouTube channel URL, @handle, or UC channel ID.'];
|
||||
}
|
||||
|
||||
$fetch = yt_http_get($url);
|
||||
if ($fetch['body'] === '' || $fetch['error']) {
|
||||
return ['url' => $url, 'videos' => [], 'error' => 'Could not fetch YouTube channel page. ' . $fetch['error']];
|
||||
}
|
||||
|
||||
$data = yt_extract_initial_data($fetch['body']);
|
||||
if (!$data) {
|
||||
return ['url' => $url, 'videos' => [], 'error' => 'Could not find ytInitialData in the scraped channel page. YouTube may have returned a consent or bot-check page.'];
|
||||
}
|
||||
|
||||
$renderers = [];
|
||||
yt_collect_renderers($data, $renderers);
|
||||
|
||||
$videos = [];
|
||||
$seen = [];
|
||||
foreach ($renderers as $renderer) {
|
||||
$video = yt_video_from_renderer($renderer);
|
||||
if (!$video || isset($seen[$video['id']])) continue;
|
||||
$seen[$video['id']] = true;
|
||||
$videos[] = $video;
|
||||
if (count($videos) >= $limit) break;
|
||||
$last_error = '';
|
||||
$found_initial_data = false;
|
||||
|
||||
foreach (yt_channel_candidate_urls($url) as $candidate_url) {
|
||||
$fetch = yt_http_get($candidate_url);
|
||||
if ($fetch['body'] === '' || $fetch['error']) {
|
||||
$last_error = 'Could not fetch YouTube channel page. ' . $fetch['error'];
|
||||
continue;
|
||||
}
|
||||
|
||||
$data = yt_extract_initial_data($fetch['body']);
|
||||
if ($data) {
|
||||
$found_initial_data = true;
|
||||
yt_collect_video_items($data, $videos, $seen, $limit);
|
||||
}
|
||||
|
||||
if (count($videos) < $limit) {
|
||||
foreach (yt_videos_from_html($fetch['body'], $limit) as $video) {
|
||||
yt_add_video($video, $videos, $seen, $limit);
|
||||
}
|
||||
}
|
||||
|
||||
if ($videos) break;
|
||||
}
|
||||
|
||||
if (!$videos) {
|
||||
return ['url' => $url, 'videos' => [], 'error' => 'No videos were found on the scraped channel videos page.'];
|
||||
if (!$found_initial_data) {
|
||||
if ($last_error !== '') return ['url' => $url, 'videos' => [], 'error' => $last_error];
|
||||
return ['url' => $url, 'videos' => [], 'error' => 'Could not find ytInitialData or watch links in the scraped page. YouTube may have returned a consent, age gate, or bot-check page.'];
|
||||
}
|
||||
return ['url' => $url, 'videos' => [], 'error' => 'No importable videos were found. The channel may have no public videos, or YouTube may have served an unsupported layout.'];
|
||||
}
|
||||
|
||||
return ['url' => $url, 'videos' => $videos, 'error' => ''];
|
||||
|
||||
Reference in New Issue
Block a user