From 0f262cab3f67a950d9e4ec51cdc31e2fc180ae53 Mon Sep 17 00:00:00 2001 From: Ty Clifford Date: Wed, 24 Jun 2026 12:55:55 -0400 Subject: [PATCH] =?UTF-8?q?-=20Improved=20the=20YouTube=20importer=20in=20?= =?UTF-8?q?[admin/youtube=5Fimport.php=20(line=20154)](/Users/tyemecliffor?= =?UTF-8?q?d/Documents/GH/mediaplayer/admin/youtube=5Fimport.php:154).=20I?= =?UTF-8?q?t=20now=20handles=20more=20YouTube=20layouts:=20Classic=20video?= =?UTF-8?q?Renderer,=20gridVideoRenderer,=20and=20compactVideoRenderer=20S?= =?UTF-8?q?horts-style=20reelItemRenderer=20Newer=20lockupViewModel=20chan?= =?UTF-8?q?nel=20grid=20objects=20Raw=20HTML=20fallback=20by=20extracting?= =?UTF-8?q?=20watch=3Fv=3D,=20/shorts/,=20/embed/,=20and=20"videoId"=20mat?= =?UTF-8?q?ches=20A=20second=20scrape=20attempt=20using=20YouTube=E2=80=99?= =?UTF-8?q?s=20older=20grid=20query:=20view=3D0&sort=3Ddd&flow=3Dgrid=20So?= =?UTF-8?q?=20instead=20of=20stopping=20at=20=E2=80=9Cno=20videos=20found?= =?UTF-8?q?=E2=80=9D=20when=20YouTube=20changes=20the=20JSON=20shape,=20it?= =?UTF-8?q?=20now=20tries=20multiple=20extraction=20strategies=20and=20can?= =?UTF-8?q?=20still=20import=20proper=20YouTube=20embed=20URLs=20with=20fa?= =?UTF-8?q?llback=20thumbnails.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- admin/youtube_import.php | 253 ++++++++++++++++++++++++++++++++++----- 1 file changed, 220 insertions(+), 33 deletions(-) diff --git a/admin/youtube_import.php b/admin/youtube_import.php index 346b6ab..6e94a2d 100644 --- a/admin/youtube_import.php +++ b/admin/youtube_import.php @@ -155,6 +155,10 @@ function yt_text($value): string { if (is_string($value)) return trim($value); if (!is_array($value)) return ''; if (isset($value['simpleText'])) return trim((string)$value['simpleText']); + if (isset($value['content'])) return trim((string)$value['content']); + if (isset($value['text'])) return trim((string)$value['text']); + if (isset($value['label'])) return trim((string)$value['label']); + if (isset($value['accessibilityLabel'])) return trim((string)$value['accessibilityLabel']); if (isset($value['runs']) && is_array($value['runs'])) { $parts = []; foreach ($value['runs'] as $run) { @@ -168,6 +172,12 @@ function yt_text($value): string { return ''; } +function yt_video_id(string $value): string { + $value = trim($value); + if (preg_match('/^[A-Za-z0-9_-]{11}$/', $value)) return $value; + return ''; +} + function yt_best_thumbnail(array $renderer): string { $thumbs = $renderer['thumbnail']['thumbnails'] ?? []; if (!is_array($thumbs) || empty($thumbs)) return ''; @@ -175,6 +185,36 @@ function yt_best_thumbnail(array $renderer): string { return isset($thumbs[0]['url']) ? html_entity_decode((string)$thumbs[0]['url'], ENT_QUOTES | ENT_HTML5, 'UTF-8') : ''; } +function yt_collect_thumbnail_candidates($node, array &$candidates): void { + if (!is_array($node)) return; + + if (isset($node['url']) && is_string($node['url'])) { + $url = html_entity_decode($node['url'], ENT_QUOTES | ENT_HTML5, 'UTF-8'); + if (preg_match('#https?://[^"\']*(?:ytimg|ggpht)[^"\']*#i', $url)) { + $candidates[] = [ + 'url' => $url, + 'width' => (int)($node['width'] ?? 0), + 'score' => str_contains($url, 'i.ytimg.com') ? 2 : 1, + ]; + } + } + + foreach ($node as $value) { + if (is_array($value)) yt_collect_thumbnail_candidates($value, $candidates); + } +} + +function yt_best_thumbnail_any(array $node, string $video_id = ''): string { + $candidates = []; + yt_collect_thumbnail_candidates($node, $candidates); + if ($candidates) { + usort($candidates, fn($a, $b) => (($b['score'] <=> $a['score']) ?: ($b['width'] <=> $a['width']))); + return $candidates[0]['url']; + } + + return $video_id !== '' ? 'https://i.ytimg.com/vi/' . rawurlencode($video_id) . '/hqdefault.jpg' : ''; +} + function yt_duration_seconds(string $duration): int { $duration = trim($duration); if ($duration === '' || stripos($duration, 'live') !== false) return 0; @@ -202,17 +242,113 @@ function yt_renderer_duration(array $renderer): string { return ''; } -function yt_collect_renderers($node, array &$renderers): void { - if (!is_array($node)) return; +function yt_video_record(string $id, string $title, string $description = '', string $duration_text = '', string $thumbnail_url = ''): ?array { + $id = yt_video_id($id); + $title = trim($title); + if ($id === '') return null; + if ($title === '') $title = 'YouTube Video ' . $id; - foreach (['videoRenderer', 'gridVideoRenderer'] as $key) { + return [ + 'id' => $id, + 'title' => $title, + 'description' => trim($description), + 'duration' => yt_duration_seconds($duration_text), + 'duration_text' => trim($duration_text), + 'thumbnail_url' => $thumbnail_url ?: 'https://i.ytimg.com/vi/' . rawurlencode($id) . '/hqdefault.jpg', + 'watch_url' => 'https://www.youtube.com/watch?v=' . rawurlencode($id), + ]; +} + +function yt_string_at_path(array $node, array $path): string { + $current = $node; + foreach ($path as $key) { + if (!is_array($current) || !array_key_exists($key, $current)) return ''; + $current = $current[$key]; + } + return yt_text($current); +} + +function yt_find_first_text_by_key($node, array $keys): string { + if (!is_array($node)) return ''; + foreach ($keys as $key) { + if (isset($node[$key])) { + $text = yt_text($node[$key]); + if ($text !== '') return $text; + } + } + foreach ($node as $value) { + if (is_array($value)) { + $text = yt_find_first_text_by_key($value, $keys); + if ($text !== '') return $text; + } + } + return ''; +} + +function yt_video_from_reel_renderer(array $renderer): ?array { + $id = yt_video_id((string)($renderer['videoId'] ?? '')); + if ($id === '') return null; + $title = yt_text($renderer['headline'] ?? []); + if ($title === '') $title = yt_text($renderer['accessibility']['accessibilityData']['label'] ?? []); + if ($title === '') $title = yt_find_first_text_by_key($renderer, ['title', 'headline']); + $views = yt_text($renderer['viewCountText'] ?? []); + $description = $views !== '' ? $views : ''; + return yt_video_record($id, $title, $description, '', yt_best_thumbnail_any($renderer, $id)); +} + +function yt_video_from_lockup_model(array $model): ?array { + $id = yt_video_id((string)($model['contentId'] ?? $model['videoId'] ?? '')); + if ($id === '') { + $url = yt_find_first_text_by_key($model, ['url', 'webCommandMetadata']); + if (preg_match('/(?:watch\?v=|shorts\/)([A-Za-z0-9_-]{11})/', $url, $m)) $id = $m[1]; + } + if ($id === '') return null; + + $title = yt_string_at_path($model, ['metadata', 'lockupMetadataViewModel', 'title']); + if ($title === '') $title = yt_find_first_text_by_key($model, ['title', 'headline', 'contentTitle']); + + $metadata = []; + foreach (['metadata', 'subtitle', 'secondarySubtitle', 'viewCountText', 'publishedTimeText'] as $key) { + $text = yt_find_first_text_by_key($model[$key] ?? [], ['content', 'simpleText', 'text']); + if ($text !== '') $metadata[] = $text; + } + + $duration = yt_find_first_text_by_key($model, ['duration', 'lengthText', 'thumbnailOverlayTimeStatusRenderer']); + if ($duration !== '' && !preg_match('/\d/', $duration)) $duration = ''; + + return yt_video_record($id, $title, implode("\n", array_unique($metadata)), $duration, yt_best_thumbnail_any($model, $id)); +} + +function yt_add_video(array $video, array &$videos, array &$seen, int $limit): void { + if (isset($seen[$video['id']]) || count($videos) >= $limit) return; + $seen[$video['id']] = true; + $videos[] = $video; +} + +function yt_collect_video_items($node, array &$videos, array &$seen, int $limit): void { + if (!is_array($node)) return; + if (count($videos) >= $limit) return; + + foreach (['videoRenderer', 'gridVideoRenderer', 'compactVideoRenderer'] as $key) { if (isset($node[$key]) && is_array($node[$key]) && !empty($node[$key]['videoId'])) { - $renderers[] = $node[$key]; + $video = yt_video_from_renderer($node[$key]); + if ($video) yt_add_video($video, $videos, $seen, $limit); } } + if (isset($node['reelItemRenderer']) && is_array($node['reelItemRenderer'])) { + $video = yt_video_from_reel_renderer($node['reelItemRenderer']); + if ($video) yt_add_video($video, $videos, $seen, $limit); + } + + if (isset($node['lockupViewModel']) && is_array($node['lockupViewModel'])) { + $video = yt_video_from_lockup_model($node['lockupViewModel']); + if ($video) yt_add_video($video, $videos, $seen, $limit); + } + foreach ($node as $value) { - if (is_array($value)) yt_collect_renderers($value, $renderers); + if (is_array($value)) yt_collect_video_items($value, $videos, $seen, $limit); + if (count($videos) >= $limit) return; } } @@ -229,15 +365,58 @@ function yt_video_from_renderer(array $renderer): ?array { $views = yt_text($renderer['viewCountText'] ?? []); $description_parts = array_filter([$snippet, trim($published . ($published && $views ? ' - ' : '') . $views)]); - return [ - 'id' => $id, - 'title' => $title, - 'description' => implode("\n", $description_parts), - 'duration' => yt_duration_seconds($duration_text), - 'duration_text' => $duration_text, - 'thumbnail_url' => yt_best_thumbnail($renderer), - 'watch_url' => 'https://www.youtube.com/watch?v=' . rawurlencode($id), + return yt_video_record($id, $title, implode("\n", $description_parts), $duration_text, yt_best_thumbnail($renderer)); +} + +function yt_title_near_video_id(string $html, string $id): string { + $quoted_id = preg_quote($id, '/'); + $patterns = [ + '/"videoId":"' . $quoted_id . '".{0,5000}?"title":\{"runs":\[\{"text":"([^"]+)"/s', + '/"videoId":"' . $quoted_id . '".{0,5000}?"title":\{"simpleText":"([^"]+)"/s', + '/"title":\{"runs":\[\{"text":"([^"]+)"\}\]\}.{0,5000}?"videoId":"' . $quoted_id . '"/s', + '/"title":\{"simpleText":"([^"]+)"\}.{0,5000}?"videoId":"' . $quoted_id . '"/s', ]; + foreach ($patterns as $pattern) { + if (preg_match($pattern, $html, $m)) { + $decoded = json_decode('"' . str_replace('"', '\"', $m[1]) . '"'); + return is_string($decoded) ? $decoded : html_entity_decode($m[1], ENT_QUOTES | ENT_HTML5, 'UTF-8'); + } + } + return ''; +} + +function yt_duration_near_video_id(string $html, string $id): string { + $quoted_id = preg_quote($id, '/'); + $patterns = [ + '/"videoId":"' . $quoted_id . '".{0,5000}?"lengthText":\{"[^}]*"simpleText":"([^"]+)"/s', + '/"lengthText":\{"[^}]*"simpleText":"([^"]+)".{0,5000}?"videoId":"' . $quoted_id . '"/s', + ]; + foreach ($patterns as $pattern) { + if (preg_match($pattern, $html, $m)) return html_entity_decode($m[1], ENT_QUOTES | ENT_HTML5, 'UTF-8'); + } + return ''; +} + +function yt_videos_from_html(string $html, int $limit): array { + preg_match_all('/(?:watch\?v=|shorts\/|embed\/|\"videoId\":\")([A-Za-z0-9_-]{11})/', $html, $matches); + $ids = array_values(array_unique($matches[1] ?? [])); + $videos = []; + $seen = []; + foreach ($ids as $id) { + if (count($videos) >= $limit) break; + $title = yt_title_near_video_id($html, $id); + $duration = yt_duration_near_video_id($html, $id); + $video = yt_video_record($id, $title, '', $duration, 'https://i.ytimg.com/vi/' . rawurlencode($id) . '/hqdefault.jpg'); + if ($video) yt_add_video($video, $videos, $seen, $limit); + } + return $videos; +} + +function yt_channel_candidate_urls(string $url): array { + return array_values(array_unique([ + $url, + media_append_query($url, ['view' => '0', 'sort' => 'dd', 'flow' => 'grid']), + ])); } function yt_scrape_channel_videos(string $input, int $limit): array { @@ -246,31 +425,39 @@ function yt_scrape_channel_videos(string $input, int $limit): array { return ['url' => '', 'videos' => [], 'error' => 'Enter a YouTube channel URL, @handle, or UC channel ID.']; } - $fetch = yt_http_get($url); - if ($fetch['body'] === '' || $fetch['error']) { - return ['url' => $url, 'videos' => [], 'error' => 'Could not fetch YouTube channel page. ' . $fetch['error']]; - } - - $data = yt_extract_initial_data($fetch['body']); - if (!$data) { - return ['url' => $url, 'videos' => [], 'error' => 'Could not find ytInitialData in the scraped channel page. YouTube may have returned a consent or bot-check page.']; - } - - $renderers = []; - yt_collect_renderers($data, $renderers); - $videos = []; $seen = []; - foreach ($renderers as $renderer) { - $video = yt_video_from_renderer($renderer); - if (!$video || isset($seen[$video['id']])) continue; - $seen[$video['id']] = true; - $videos[] = $video; - if (count($videos) >= $limit) break; + $last_error = ''; + $found_initial_data = false; + + foreach (yt_channel_candidate_urls($url) as $candidate_url) { + $fetch = yt_http_get($candidate_url); + if ($fetch['body'] === '' || $fetch['error']) { + $last_error = 'Could not fetch YouTube channel page. ' . $fetch['error']; + continue; + } + + $data = yt_extract_initial_data($fetch['body']); + if ($data) { + $found_initial_data = true; + yt_collect_video_items($data, $videos, $seen, $limit); + } + + if (count($videos) < $limit) { + foreach (yt_videos_from_html($fetch['body'], $limit) as $video) { + yt_add_video($video, $videos, $seen, $limit); + } + } + + if ($videos) break; } if (!$videos) { - return ['url' => $url, 'videos' => [], 'error' => 'No videos were found on the scraped channel videos page.']; + if (!$found_initial_data) { + if ($last_error !== '') return ['url' => $url, 'videos' => [], 'error' => $last_error]; + return ['url' => $url, 'videos' => [], 'error' => 'Could not find ytInitialData or watch links in the scraped page. YouTube may have returned a consent, age gate, or bot-check page.']; + } + return ['url' => $url, 'videos' => [], 'error' => 'No importable videos were found. The channel may have no public videos, or YouTube may have served an unsupported layout.']; } return ['url' => $url, 'videos' => $videos, 'error' => ''];