' . $this->highlightTerms($excerpt, $this->searchTerm) . '
'); - } - array_push($content, $page); - } catch (Exception $e) { - // continue + global $url; + global $WHERE_AM_I; + global $content; + + // Render with regular listing templates; many themes only know home/page. + $url->setWhereAmI('home'); + $WHERE_AM_I = 'home'; + $content = array(); + + foreach ($this->pagesFound as $pageKey) { + try { + $page = new Page($pageKey); + $excerpt = $this->getSearchExcerpt($this->pageRawContent($page), $this->searchTerm); + if (!empty($excerpt)) { + $page->setField('content', '' . $this->highlightTerms($excerpt, $this->searchTerm) . '
'); } + $content[] = $page; + } catch (Exception $e) { + $this->logSearchError('Result page failed: ' . $e->getMessage()); } } + + return true; } - /** - * Highlight search terms in plain text and return safe HTML. - */ public function highlightTerms($text, $searchTerm) { $safeText = htmlspecialchars((string) $text, ENT_QUOTES, 'UTF-8'); - if (!$this->getValue('highlightResults') || empty($searchTerm)) { return $safeText; } - $words = $this->splitWords($searchTerm); - $words = array_filter($words, function($word) { - return mb_strlen($word, 'UTF-8') >= 2; - }); - $words = array_values(array_unique($words)); - - if (empty($words)) { + $terms = $this->queryTerms($searchTerm); + if (empty($terms)) { return $safeText; } - usort($words, function($a, $b) { - return mb_strlen($b, 'UTF-8') - mb_strlen($a, 'UTF-8'); + usort($terms, function($a, $b) { + return mb_strlen($b, CHARSET) - mb_strlen($a, CHARSET); }); - $escapedWords = array_map(function($word) { - return preg_quote(htmlspecialchars($word, ENT_QUOTES, 'UTF-8'), '/'); - }, $words); - - $pattern = '/(' . implode('|', $escapedWords) . ')/iu'; + $escapedTerms = array_map(function($term) { + return preg_quote(htmlspecialchars($term, ENT_QUOTES, 'UTF-8'), '/'); + }, $terms); + $pattern = '/(' . implode('|', $escapedTerms) . ')/iu'; $highlighted = preg_replace($pattern, '$1', $safeText); - if ($highlighted === null) { - return $safeText; - } - - return $highlighted; + return ($highlighted === null) ? $safeText : $highlighted; } - /** - * Get excerpt around the search term - */ public function getSearchExcerpt($content, $searchTerm, $length = null) { - if ($length === null) { - $length = $this->getValue('excerptLength'); - } + $length = ($length === null) ? $this->getValue('excerptLength') : $length; $length = min(800, max(80, (int) $length)); - $content = $this->plainText($content); if (empty($content)) { return ''; } - $words = $this->splitWords($searchTerm); - $words = array_filter($words, function($word) { - return mb_strlen($word, 'UTF-8') >= 2; - }); - $firstPos = false; - foreach ($words as $word) { - $pos = mb_stripos($content, $word, 0, 'UTF-8'); + foreach ($this->queryTerms($searchTerm) as $term) { + $pos = mb_stripos($content, $term, 0, CHARSET); if ($pos !== false && ($firstPos === false || $pos < $firstPos)) { $firstPos = $pos; } @@ -629,23 +582,23 @@ EOF; $start = ($firstPos === false) ? 0 : max(0, $firstPos - (int) ($length / 3)); if ($start > 0) { - $prefix = mb_substr($content, 0, $start, 'UTF-8'); - $spacePos = mb_strrpos($prefix, ' ', 0, 'UTF-8'); + $prefix = mb_substr($content, 0, $start, CHARSET); + $spacePos = mb_strrpos($prefix, ' ', 0, CHARSET); if ($spacePos !== false) { $start = $spacePos + 1; } } - $excerpt = mb_substr($content, $start, $length, 'UTF-8'); - $contentLength = mb_strlen($content, 'UTF-8'); + $excerpt = mb_substr($content, $start, $length, CHARSET); + $contentLength = mb_strlen($content, CHARSET); if ($start > 0) { $excerpt = '...' . $excerpt; } if ($start + $length < $contentLength) { - $lastSpace = mb_strrpos($excerpt, ' ', 0, 'UTF-8'); + $lastSpace = mb_strrpos($excerpt, ' ', 0, CHARSET); if ($lastSpace !== false && $lastSpace > max(20, $length - 40)) { - $excerpt = mb_substr($excerpt, 0, $lastSpace, 'UTF-8'); + $excerpt = mb_substr($excerpt, 0, $lastSpace, CHARSET); } $excerpt .= '...'; } @@ -656,7 +609,7 @@ EOF; private function plainText($text) { $text = Text::removeHTMLTags((string) $text); - $text = html_entity_decode($text, ENT_QUOTES, 'UTF-8'); + $text = html_entity_decode($text, ENT_QUOTES, CHARSET); $plainText = preg_replace('/\s+/u', ' ', $text); if ($plainText === null) { $plainText = preg_replace('/\s+/', ' ', $text); @@ -668,28 +621,14 @@ EOF; return trim((string) $plainText); } - private function splitWords($text) - { - $words = preg_split('/\s+/u', trim((string) $text), -1, PREG_SPLIT_NO_EMPTY); - if (!is_array($words)) { - $words = preg_split('/\s+/', trim((string) $text), -1, PREG_SPLIT_NO_EMPTY); - } - if (!is_array($words)) { - return array(); - } - - return $words; - } - - // Generate the cache file - // This function is necessary to call it when you create, edit or remove content private function createCache() { - // Get all pages published global $pages; - global $categories; $list = $pages->getList($pageNumber = 1, $numberOfItems = -1, $published = true, $static = true, $sticky = true, $draft = false, $scheduled = false); + if (!is_array($list)) { + return false; + } $cache = array(); foreach ($list as $pageKey) { @@ -698,323 +637,320 @@ EOF; continue; } - // Process content $characterLimit = max(250, (int) $this->getValue('wordsToCachePerPage') * 6); - $contentSource = method_exists($page, 'contentRaw') ? $page->contentRaw() : $page->content(); - $content = Text::truncate($this->plainText($contentSource), $characterLimit, ''); - // Include the page to the cache $cache[$pageKey] = array( 'title' => $this->plainText($page->title()), 'description' => $this->plainText($page->description()), - 'content' => $content, + 'content' => Text::truncate($this->plainText($this->pageRawContent($page)), $characterLimit, ''), 'tags' => '', 'category' => '' ); - // Add tags if enabled if ($this->getValue('searchInTags')) { $pageTags = $page->tags(true); if (!empty($pageTags)) { - $cache[$pageKey]['tags'] = implode(' ', $pageTags); + $cache[$pageKey]['tags'] = $this->plainText(implode(' ', $pageTags)); } } - // Add category if enabled if ($this->getValue('searchInCategories')) { - $categoryKey = $page->categoryKey(); - if (!empty($categoryKey) && isset($categories)) { - try { - $category = new Category($categoryKey); - $cache[$pageKey]['category'] = $this->plainText($category->name()); - } catch (Exception $e) { - // Category not found - } - } + $cache[$pageKey]['category'] = $this->plainText($page->category()); } } - // Generate JSON file with the cache + $workspace = $this->workspace(); + if (!is_dir($workspace)) { + $permissions = defined('DIR_PERMISSIONS') ? DIR_PERMISSIONS : 0755; + mkdir($workspace, $permissions, true); + } + $json = json_encode($cache, JSON_UNESCAPED_UNICODE); return file_put_contents($this->cacheFile(), $json, LOCK_EX); } - // Returns the absolute path of the cache file private function cacheFile() { return $this->workspace() . 'cache.json'; } - // Search text inside the cache - // Returns an array with the pages keys related to the text - // The array is sorted by score private function search($text) { - $text = trim($text); - - // Check minimum characters - $minChars = max(1, (int) $this->getValue('minChars')); - if (mb_strlen($text, 'UTF-8') < $minChars) { + $query = $this->normalizeQuery($text); + if (mb_strlen($query, CHARSET) < $this->minChars()) { return array(); } - // Read the cache file - $cacheFile = $this->cacheFile(); - if (!file_exists($cacheFile)) { - $this->createCache(); - } - - $json = file_get_contents($cacheFile); - $cache = json_decode($json, true); - - if (!is_array($cache)) { - $this->createCache(); - $json = file_get_contents($cacheFile); - $cache = json_decode($json, true); - } - - if (empty($cache) || !is_array($cache)) { - return array(); - } - - if ($this->cacheContainsStoredSearchFields($cache)) { - $this->createCache(); - $json = file_get_contents($cacheFile); - $cache = json_decode($json, true); - if (empty($cache) || !is_array($cache)) { - return array(); - } - } - - $terms = $this->tokenizeSearchTerm($text); - if (empty($terms)) { + $cache = $this->loadCache(); + $terms = $this->queryTerms($query); + if (empty($cache) || empty($terms)) { return array(); } $scores = array(); - foreach ($cache as $pageKey => $data) { - if (!is_array($data)) { - continue; - } - - $score = $this->scoreSearchResult($data, $text, $terms); + foreach ($cache as $pageKey => $pageData) { + $score = $this->scorePage($pageData, $query, $terms); if ($score > 0) { $scores[$pageKey] = $score; } } + if (empty($scores)) { + return array(); + } + arsort($scores, SORT_NUMERIC); return array_keys($scores); } - private function cacheContainsStoredSearchFields($cache) + private function loadCache() { - foreach ($cache as $data) { - return is_array($data) && isset($data['search']); + $cacheFile = $this->cacheFile(); + if (!file_exists($cacheFile)) { + $this->createCache(); + } + + $cache = $this->readCache(); + if (!is_array($cache) || $this->cacheIsLegacy($cache)) { + $this->createCache(); + $cache = $this->readCache(); + } + + return is_array($cache) ? $cache : array(); + } + + private function readCache() + { + $cacheFile = $this->cacheFile(); + if (!file_exists($cacheFile)) { + return array(); + } + + $json = file_get_contents($cacheFile); + if ($json === false || $json === '') { + return array(); + } + + $cache = json_decode($json, true); + return is_array($cache) ? $cache : array(); + } + + private function cacheIsLegacy($cache) + { + foreach ($cache as $pageData) { + if (is_array($pageData) && isset($pageData['search'])) { + return true; + } + return false; } return false; } - private function buildSearchFields($data) + private function scorePage($pageData, $query, $terms) { - $fields = array( - 'title' => $this->normalizeSearchText(isset($data['title']) ? $data['title'] : ''), - 'description' => $this->normalizeSearchText(isset($data['description']) ? $data['description'] : ''), - 'content' => $this->normalizeSearchText(isset($data['content']) ? $data['content'] : ''), - 'tags' => $this->normalizeSearchText(isset($data['tags']) ? $data['tags'] : ''), - 'category' => $this->normalizeSearchText(isset($data['category']) ? $data['category'] : '') - ); - $fields['all'] = trim(implode(' ', $fields)); - - return $fields; - } - - private function searchableFields($data) - { - if (isset($data['search']) && is_array($data['search'])) { - return $data['search'] + array( - 'title' => '', - 'description' => '', - 'content' => '', - 'tags' => '', - 'category' => '', - 'all' => '' - ); + if (!is_array($pageData)) { + return 0; } - return $this->buildSearchFields($data); + $fields = array( + 'title' => $this->normalizeSearchText($this->arrayValue($pageData, 'title')), + 'tags' => $this->normalizeSearchText($this->arrayValue($pageData, 'tags')), + 'category' => $this->normalizeSearchText($this->arrayValue($pageData, 'category')), + 'description' => $this->normalizeSearchText($this->arrayValue($pageData, 'description')), + 'content' => $this->normalizeSearchText($this->arrayValue($pageData, 'content')) + ); + + $weights = array( + 'title' => 100, + 'tags' => 80, + 'category' => 70, + 'description' => 45, + 'content' => 20 + ); + $score = 0; + $matches = 0; + + foreach ($fields as $fieldName => $fieldText) { + if ($fieldText === '') { + continue; + } + + if (strpos($fieldText, $query) !== false) { + $score += $weights[$fieldName] * 3; + } + + foreach ($terms as $term) { + $termScore = $this->scoreTerm($fieldText, $term, $weights[$fieldName]); + if ($termScore > 0) { + $score += $termScore; + $matches++; + } + } + } + + if ($matches === 0) { + $score += $this->fuzzyTitleScore($fields, $terms); + } + + if (count($terms) > 1 && $matches >= count($terms)) { + $score += 40; + } + + return $score; + } + + private function scoreTerm($fieldText, $term, $weight) + { + if ($fieldText === '' || $term === '') { + return 0; + } + + $quoted = preg_quote($term, '/'); + $wholeWordMatches = preg_match_all('/(^| )' . $quoted . '(?= |$)/u', $fieldText, $matches); + if ($wholeWordMatches > 0) { + return $weight + min(50, $wholeWordMatches * 8); + } + + if (mb_strlen($term, CHARSET) >= 4 && strpos($fieldText, $term) !== false) { + return (int) ($weight * 0.45); + } + + return 0; + } + + private function fuzzyTitleScore($fields, $terms) + { + $text = trim($fields['title'] . ' ' . $fields['tags'] . ' ' . $fields['category']); + if ($text === '') { + return 0; + } + + $words = $this->splitWords($text); + if (empty($words)) { + return 0; + } + + $score = 0; + foreach ($terms as $term) { + if (mb_strlen($term, CHARSET) < 4) { + continue; + } + $allowedDistance = (mb_strlen($term, CHARSET) <= 5) ? 1 : 2; + foreach ($words as $word) { + if (abs(strlen($word) - strlen($term)) > $allowedDistance) { + continue; + } + if (levenshtein($term, $word) <= $allowedDistance) { + $score += 30; + break; + } + } + } + + return $score; + } + + private function normalizeQuery($text) + { + $text = $this->plainText(urldecode((string) $text)); + if (mb_strlen($text, CHARSET) > 120) { + $text = mb_substr($text, 0, 120, CHARSET); + } + + return $text; + } + + private function searchTermFromUrl() + { + $term = $this->webhook('search', true, false); + if (!is_string($term)) { + return ''; + } + + return $this->normalizeQuery(trim($term, '/')); + } + + private function queryTerms($query) + { + $normalized = $this->normalizeSearchText($query); + $terms = array(); + foreach ($this->splitWords($normalized) as $word) { + if (mb_strlen($word, CHARSET) >= 2) { + $terms[$word] = true; + } + } + + return array_keys($terms); + } + + private function splitWords($text) + { + $words = preg_split('/\s+/u', trim((string) $text), -1, PREG_SPLIT_NO_EMPTY); + if (!is_array($words)) { + $words = preg_split('/\s+/', trim((string) $text), -1, PREG_SPLIT_NO_EMPTY); + } + + return is_array($words) ? $words : array(); } private function normalizeSearchText($text) { $text = $this->plainText($text); - + $originalText = $text; if (function_exists('transliterator_transliterate')) { $transliterated = @transliterator_transliterate('Any-Latin; Latin-ASCII;', $text); if ($transliterated !== false) { $text = $transliterated; } } elseif (function_exists('iconv')) { - $transliterated = @iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $text); + $transliterated = @iconv(CHARSET, 'ASCII//TRANSLIT//IGNORE', $text); if ($transliterated !== false) { $text = $transliterated; } } - $text = mb_strtolower($text, 'UTF-8'); - $normalized = preg_replace('/[^\p{L}\p{N}]+/u', ' ', $text); - if ($normalized === null) { - $normalized = preg_replace('/[^a-z0-9]+/i', ' ', $text); + $text = mb_strtolower($text, CHARSET); + $text = preg_replace('/[^\p{L}\p{N}]+/u', ' ', $text); + if ($text === null) { + $text = preg_replace('/[^a-z0-9]+/i', ' ', $originalText); } - if ($normalized === null) { - $normalized = $text; - } - $collapsed = preg_replace('/\s+/u', ' ', $normalized); - if ($collapsed === null) { - $collapsed = preg_replace('/\s+/', ' ', $normalized); - } - if ($collapsed === null) { - $collapsed = $normalized; + if ($text === null) { + return ''; } - return trim((string) $collapsed); + $text = preg_replace('/\s+/u', ' ', $text); + return trim((string) $text); } - private function tokenizeSearchTerm($text) + private function pageRawContent($page) { - $normalized = $this->normalizeSearchText($text); - $words = $this->splitWords($normalized); - - $tokens = array(); - foreach ($words as $word) { - if (mb_strlen($word, 'UTF-8') >= 2) { - $tokens[$word] = true; - } + if (method_exists($page, 'contentRaw')) { + return $page->contentRaw(); } - return array_keys($tokens); + return $page->content(); } - private function scoreSearchResult($data, $query, $terms) + private function minChars() { - $fields = $this->searchableFields($data); - $query = $this->normalizeSearchText($query); - $score = 0; - - if (!empty($query)) { - $phraseWeights = array( - 'title' => 220, - 'tags' => 180, - 'category' => 160, - 'description' => 110, - 'content' => 70 - ); - - foreach ($phraseWeights as $field => $weight) { - if (!empty($fields[$field]) && strpos($fields[$field], $query) !== false) { - $score += $weight + min(60, $this->countOccurrences($fields[$field], $query) * 8); - } - } - } - - $matchedTerms = 0; - foreach ($terms as $term) { - $termScore = 0; - $termScore += $this->scoreTermInField($fields['title'], $term, 95); - $termScore += $this->scoreTermInField($fields['tags'], $term, 80); - $termScore += $this->scoreTermInField($fields['category'], $term, 75); - $termScore += $this->scoreTermInField($fields['description'], $term, 45); - $termScore += $this->scoreTermInField($fields['content'], $term, 25); - - if ($termScore === 0) { - $termScore += $this->fuzzyTermScore($term, trim($fields['title'] . ' ' . $fields['tags'] . ' ' . $fields['category'] . ' ' . $fields['description']), 24); - } - - if ($termScore > 0) { - $matchedTerms++; - $score += $termScore; - } - } - - if ($matchedTerms === 0) { - return 0; - } - - if (count($terms) > 1) { - if ($matchedTerms === count($terms)) { - $score += 45; - } else { - $score -= (count($terms) - $matchedTerms) * 25; - } - } - - return max(0, $score); + return max(1, (int) $this->getValue('minChars')); } - private function scoreTermInField($field, $term, $weight) + private function isSearchRequest() { - if (empty($field)) { - return 0; - } - - $quoted = preg_quote($term, '/'); - $fullMatches = preg_match_all('/(^| )' . $quoted . '(?= |$)/u', $field, $matches); - if ($fullMatches > 0) { - return $weight + min(40, $fullMatches * 6); - } - - if (mb_strlen($term, 'UTF-8') >= 4 && strpos($field, $term) !== false) { - return (int) ($weight * 0.55) + min(25, $this->countOccurrences($field, $term) * 4); - } - - return 0; + return $this->webhook('search', false, false); } - private function countOccurrences($field, $needle) + private function arrayValue($array, $key) { - if ($needle === '') { - return 0; - } - - return substr_count($field, $needle); + return (is_array($array) && isset($array[$key])) ? $array[$key] : ''; } - private function fuzzyTermScore($term, $field, $weight) + private function logSearchError($message) { - if (mb_strlen($term, 'UTF-8') < 4 || empty($field)) { - return 0; + if (class_exists('Log') && defined('LOG_SEP')) { + Log::set(__METHOD__ . LOG_SEP . $message); } - - $words = $this->splitWords($field); - - $bestScore = 0; - $checked = array(); - $checkedCount = 0; - $termLength = strlen($term); - $allowedDistance = ($termLength <= 5) ? 1 : 2; - - foreach ($words as $word) { - if (isset($checked[$word])) { - continue; - } - $checked[$word] = true; - - if (strlen($word) > 64 || abs(strlen($word) - $termLength) > $allowedDistance) { - continue; - } - - $checkedCount++; - if ($checkedCount > 250) { - break; - } - - $distance = levenshtein($term, $word); - if ($distance <= $allowedDistance) { - $bestScore = max($bestScore, $weight - ($distance * 5)); - } - } - - return $bestScore; } }