- Rewrote the search internals in [plugin.php](/Users/tyemeclifford/Documents/GH/blog/bl-plugins/search/plugin.php).
What changed: Replaced the cache/search/scoring flow with a simpler weighted search. Search cache now stores only plain page fields: title, description, raw content, tags, category. Removed the persisted normalized search payloads and legacy helper paths. Search renders through normal listing templates for theme compatibility. Added fail-open behavior: unexpected search errors log and return no results instead of blanking the page. Kept excerpts and highlighting, but made them plain-text based and safer.
This commit is contained in:
Ty Clifford
+299
-363
@@ -463,65 +463,42 @@ EOF;
|
|||||||
|
|
||||||
public function beforeAll()
|
public function beforeAll()
|
||||||
{
|
{
|
||||||
// Check if the URL match with the webhook
|
if (!$this->isSearchRequest()) {
|
||||||
$webhook = 'search';
|
return false;
|
||||||
if ($this->webhook($webhook, false, false)) {
|
|
||||||
global $site;
|
|
||||||
global $url;
|
|
||||||
|
|
||||||
// Change the whereAmI to avoid load pages in the rule 69.pages
|
|
||||||
// This is only for performance purpose
|
|
||||||
$url->setWhereAmI('search');
|
|
||||||
|
|
||||||
// Get the string to search from the URL
|
|
||||||
$stringToSearch = $this->webhook($webhook, true, false);
|
|
||||||
$stringToSearch = trim($stringToSearch, '/');
|
|
||||||
$stringToSearch = urldecode($stringToSearch);
|
|
||||||
$normalizedWhitespace = preg_replace('/\s+/u', ' ', $stringToSearch);
|
|
||||||
if ($normalizedWhitespace === null) {
|
|
||||||
$normalizedWhitespace = preg_replace('/\s+/', ' ', $stringToSearch);
|
|
||||||
}
|
|
||||||
if ($normalizedWhitespace !== null) {
|
|
||||||
$stringToSearch = $normalizedWhitespace;
|
|
||||||
}
|
|
||||||
$stringToSearch = trim((string) $stringToSearch);
|
|
||||||
if (mb_strlen($stringToSearch, 'UTF-8') > 120) {
|
|
||||||
$stringToSearch = mb_substr($stringToSearch, 0, 120, 'UTF-8');
|
|
||||||
}
|
|
||||||
$this->searchTerm = $stringToSearch;
|
|
||||||
|
|
||||||
// Search the string in the cache and get all pages with matches
|
|
||||||
try {
|
|
||||||
$list = $this->search($stringToSearch);
|
|
||||||
} catch (Exception $e) {
|
|
||||||
if (class_exists('Log')) {
|
|
||||||
Log::set(__METHOD__ . LOG_SEP . 'Search failed: ' . $e->getMessage());
|
|
||||||
}
|
|
||||||
$list = array();
|
|
||||||
}
|
|
||||||
$this->numberOfItems = count($list);
|
|
||||||
|
|
||||||
// Split the content in pages
|
|
||||||
// The first page number is 1, so the real is 0
|
|
||||||
$realPageNumber = $url->pageNumber() - 1;
|
|
||||||
$itemsPerPage = $site->itemsPerPage();
|
|
||||||
if ($itemsPerPage <= 0) {
|
|
||||||
if ($realPageNumber === 0) {
|
|
||||||
$this->pagesFound = $list;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
$chunks = array_chunk($list, $itemsPerPage);
|
|
||||||
if (isset($chunks[$realPageNumber])) {
|
|
||||||
$this->pagesFound = $chunks[$realPageNumber];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
global $site;
|
||||||
|
global $url;
|
||||||
|
|
||||||
|
// Stop the normal page builder from treating /search/... as a page.
|
||||||
|
$url->setWhereAmI('search');
|
||||||
|
|
||||||
|
$this->searchTerm = $this->searchTermFromUrl();
|
||||||
|
try {
|
||||||
|
$list = $this->search($this->searchTerm);
|
||||||
|
} catch (Throwable $e) {
|
||||||
|
$this->logSearchError('Search failed: ' . $e->getMessage());
|
||||||
|
$list = array();
|
||||||
|
}
|
||||||
|
$this->numberOfItems = count($list);
|
||||||
|
|
||||||
|
$pageNumber = max(1, (int) $url->pageNumber());
|
||||||
|
$itemsPerPage = max(0, (int) $site->itemsPerPage());
|
||||||
|
|
||||||
|
if ($itemsPerPage === 0) {
|
||||||
|
$this->pagesFound = ($pageNumber === 1) ? $list : array();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
$offset = ($pageNumber - 1) * $itemsPerPage;
|
||||||
|
$this->pagesFound = array_slice($list, $offset, $itemsPerPage);
|
||||||
|
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function paginator()
|
public function paginator()
|
||||||
{
|
{
|
||||||
$webhook = 'search';
|
if ($this->isSearchRequest()) {
|
||||||
if ($this->webhook($webhook, false, false)) {
|
|
||||||
// Get the pre-defined variable from the rule 99.paginator.php
|
// Get the pre-defined variable from the rule 99.paginator.php
|
||||||
// Is necessary to change this variable to fit the paginator with the result from the search
|
// Is necessary to change this variable to fit the paginator with the result from the search
|
||||||
global $numberOfItems;
|
global $numberOfItems;
|
||||||
@@ -531,96 +508,72 @@ EOF;
|
|||||||
|
|
||||||
public function beforeSiteLoad()
|
public function beforeSiteLoad()
|
||||||
{
|
{
|
||||||
$webhook = 'search';
|
if (!$this->isSearchRequest()) {
|
||||||
if ($this->webhook($webhook, false, false)) {
|
return false;
|
||||||
global $url;
|
}
|
||||||
global $WHERE_AM_I;
|
|
||||||
// Feed search results through regular listing templates for theme compatibility.
|
|
||||||
$url->setWhereAmI('home');
|
|
||||||
$WHERE_AM_I = 'home';
|
|
||||||
|
|
||||||
// Get the pre-defined variable from the rule 69.pages.php
|
global $url;
|
||||||
// We change the content to show in the website
|
global $WHERE_AM_I;
|
||||||
global $content;
|
global $content;
|
||||||
$content = array();
|
|
||||||
foreach ($this->pagesFound as $pageKey) {
|
// Render with regular listing templates; many themes only know home/page.
|
||||||
try {
|
$url->setWhereAmI('home');
|
||||||
$page = new Page($pageKey);
|
$WHERE_AM_I = 'home';
|
||||||
$contentSource = method_exists($page, 'contentRaw') ? $page->contentRaw() : $page->content();
|
$content = array();
|
||||||
$excerpt = $this->getSearchExcerpt($contentSource, $this->searchTerm);
|
|
||||||
if (!empty($excerpt)) {
|
foreach ($this->pagesFound as $pageKey) {
|
||||||
$page->setField('content', '<p class="search-result-excerpt">' . $this->highlightTerms($excerpt, $this->searchTerm) . '</p>');
|
try {
|
||||||
}
|
$page = new Page($pageKey);
|
||||||
array_push($content, $page);
|
$excerpt = $this->getSearchExcerpt($this->pageRawContent($page), $this->searchTerm);
|
||||||
} catch (Exception $e) {
|
if (!empty($excerpt)) {
|
||||||
// continue
|
$page->setField('content', '<p class="search-result-excerpt">' . $this->highlightTerms($excerpt, $this->searchTerm) . '</p>');
|
||||||
}
|
}
|
||||||
|
$content[] = $page;
|
||||||
|
} catch (Exception $e) {
|
||||||
|
$this->logSearchError('Result page failed: ' . $e->getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Highlight search terms in plain text and return safe HTML.
|
|
||||||
*/
|
|
||||||
public function highlightTerms($text, $searchTerm)
|
public function highlightTerms($text, $searchTerm)
|
||||||
{
|
{
|
||||||
$safeText = htmlspecialchars((string) $text, ENT_QUOTES, 'UTF-8');
|
$safeText = htmlspecialchars((string) $text, ENT_QUOTES, 'UTF-8');
|
||||||
|
|
||||||
if (!$this->getValue('highlightResults') || empty($searchTerm)) {
|
if (!$this->getValue('highlightResults') || empty($searchTerm)) {
|
||||||
return $safeText;
|
return $safeText;
|
||||||
}
|
}
|
||||||
|
|
||||||
$words = $this->splitWords($searchTerm);
|
$terms = $this->queryTerms($searchTerm);
|
||||||
$words = array_filter($words, function($word) {
|
if (empty($terms)) {
|
||||||
return mb_strlen($word, 'UTF-8') >= 2;
|
|
||||||
});
|
|
||||||
$words = array_values(array_unique($words));
|
|
||||||
|
|
||||||
if (empty($words)) {
|
|
||||||
return $safeText;
|
return $safeText;
|
||||||
}
|
}
|
||||||
|
|
||||||
usort($words, function($a, $b) {
|
usort($terms, function($a, $b) {
|
||||||
return mb_strlen($b, 'UTF-8') - mb_strlen($a, 'UTF-8');
|
return mb_strlen($b, CHARSET) - mb_strlen($a, CHARSET);
|
||||||
});
|
});
|
||||||
|
|
||||||
$escapedWords = array_map(function($word) {
|
$escapedTerms = array_map(function($term) {
|
||||||
return preg_quote(htmlspecialchars($word, ENT_QUOTES, 'UTF-8'), '/');
|
return preg_quote(htmlspecialchars($term, ENT_QUOTES, 'UTF-8'), '/');
|
||||||
}, $words);
|
}, $terms);
|
||||||
|
|
||||||
$pattern = '/(' . implode('|', $escapedWords) . ')/iu';
|
|
||||||
|
|
||||||
|
$pattern = '/(' . implode('|', $escapedTerms) . ')/iu';
|
||||||
$highlighted = preg_replace($pattern, '<mark class="search-highlight">$1</mark>', $safeText);
|
$highlighted = preg_replace($pattern, '<mark class="search-highlight">$1</mark>', $safeText);
|
||||||
if ($highlighted === null) {
|
return ($highlighted === null) ? $safeText : $highlighted;
|
||||||
return $safeText;
|
|
||||||
}
|
|
||||||
|
|
||||||
return $highlighted;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get excerpt around the search term
|
|
||||||
*/
|
|
||||||
public function getSearchExcerpt($content, $searchTerm, $length = null)
|
public function getSearchExcerpt($content, $searchTerm, $length = null)
|
||||||
{
|
{
|
||||||
if ($length === null) {
|
$length = ($length === null) ? $this->getValue('excerptLength') : $length;
|
||||||
$length = $this->getValue('excerptLength');
|
|
||||||
}
|
|
||||||
$length = min(800, max(80, (int) $length));
|
$length = min(800, max(80, (int) $length));
|
||||||
|
|
||||||
$content = $this->plainText($content);
|
$content = $this->plainText($content);
|
||||||
if (empty($content)) {
|
if (empty($content)) {
|
||||||
return '';
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
$words = $this->splitWords($searchTerm);
|
|
||||||
$words = array_filter($words, function($word) {
|
|
||||||
return mb_strlen($word, 'UTF-8') >= 2;
|
|
||||||
});
|
|
||||||
|
|
||||||
$firstPos = false;
|
$firstPos = false;
|
||||||
foreach ($words as $word) {
|
foreach ($this->queryTerms($searchTerm) as $term) {
|
||||||
$pos = mb_stripos($content, $word, 0, 'UTF-8');
|
$pos = mb_stripos($content, $term, 0, CHARSET);
|
||||||
if ($pos !== false && ($firstPos === false || $pos < $firstPos)) {
|
if ($pos !== false && ($firstPos === false || $pos < $firstPos)) {
|
||||||
$firstPos = $pos;
|
$firstPos = $pos;
|
||||||
}
|
}
|
||||||
@@ -629,23 +582,23 @@ EOF;
|
|||||||
$start = ($firstPos === false) ? 0 : max(0, $firstPos - (int) ($length / 3));
|
$start = ($firstPos === false) ? 0 : max(0, $firstPos - (int) ($length / 3));
|
||||||
|
|
||||||
if ($start > 0) {
|
if ($start > 0) {
|
||||||
$prefix = mb_substr($content, 0, $start, 'UTF-8');
|
$prefix = mb_substr($content, 0, $start, CHARSET);
|
||||||
$spacePos = mb_strrpos($prefix, ' ', 0, 'UTF-8');
|
$spacePos = mb_strrpos($prefix, ' ', 0, CHARSET);
|
||||||
if ($spacePos !== false) {
|
if ($spacePos !== false) {
|
||||||
$start = $spacePos + 1;
|
$start = $spacePos + 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$excerpt = mb_substr($content, $start, $length, 'UTF-8');
|
$excerpt = mb_substr($content, $start, $length, CHARSET);
|
||||||
$contentLength = mb_strlen($content, 'UTF-8');
|
$contentLength = mb_strlen($content, CHARSET);
|
||||||
|
|
||||||
if ($start > 0) {
|
if ($start > 0) {
|
||||||
$excerpt = '...' . $excerpt;
|
$excerpt = '...' . $excerpt;
|
||||||
}
|
}
|
||||||
if ($start + $length < $contentLength) {
|
if ($start + $length < $contentLength) {
|
||||||
$lastSpace = mb_strrpos($excerpt, ' ', 0, 'UTF-8');
|
$lastSpace = mb_strrpos($excerpt, ' ', 0, CHARSET);
|
||||||
if ($lastSpace !== false && $lastSpace > max(20, $length - 40)) {
|
if ($lastSpace !== false && $lastSpace > max(20, $length - 40)) {
|
||||||
$excerpt = mb_substr($excerpt, 0, $lastSpace, 'UTF-8');
|
$excerpt = mb_substr($excerpt, 0, $lastSpace, CHARSET);
|
||||||
}
|
}
|
||||||
$excerpt .= '...';
|
$excerpt .= '...';
|
||||||
}
|
}
|
||||||
@@ -656,7 +609,7 @@ EOF;
|
|||||||
private function plainText($text)
|
private function plainText($text)
|
||||||
{
|
{
|
||||||
$text = Text::removeHTMLTags((string) $text);
|
$text = Text::removeHTMLTags((string) $text);
|
||||||
$text = html_entity_decode($text, ENT_QUOTES, 'UTF-8');
|
$text = html_entity_decode($text, ENT_QUOTES, CHARSET);
|
||||||
$plainText = preg_replace('/\s+/u', ' ', $text);
|
$plainText = preg_replace('/\s+/u', ' ', $text);
|
||||||
if ($plainText === null) {
|
if ($plainText === null) {
|
||||||
$plainText = preg_replace('/\s+/', ' ', $text);
|
$plainText = preg_replace('/\s+/', ' ', $text);
|
||||||
@@ -668,28 +621,14 @@ EOF;
|
|||||||
return trim((string) $plainText);
|
return trim((string) $plainText);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function splitWords($text)
|
|
||||||
{
|
|
||||||
$words = preg_split('/\s+/u', trim((string) $text), -1, PREG_SPLIT_NO_EMPTY);
|
|
||||||
if (!is_array($words)) {
|
|
||||||
$words = preg_split('/\s+/', trim((string) $text), -1, PREG_SPLIT_NO_EMPTY);
|
|
||||||
}
|
|
||||||
if (!is_array($words)) {
|
|
||||||
return array();
|
|
||||||
}
|
|
||||||
|
|
||||||
return $words;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate the cache file
|
|
||||||
// This function is necessary to call it when you create, edit or remove content
|
|
||||||
private function createCache()
|
private function createCache()
|
||||||
{
|
{
|
||||||
// Get all pages published
|
|
||||||
global $pages;
|
global $pages;
|
||||||
global $categories;
|
|
||||||
|
|
||||||
$list = $pages->getList($pageNumber = 1, $numberOfItems = -1, $published = true, $static = true, $sticky = true, $draft = false, $scheduled = false);
|
$list = $pages->getList($pageNumber = 1, $numberOfItems = -1, $published = true, $static = true, $sticky = true, $draft = false, $scheduled = false);
|
||||||
|
if (!is_array($list)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
$cache = array();
|
$cache = array();
|
||||||
foreach ($list as $pageKey) {
|
foreach ($list as $pageKey) {
|
||||||
@@ -698,323 +637,320 @@ EOF;
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process content
|
|
||||||
$characterLimit = max(250, (int) $this->getValue('wordsToCachePerPage') * 6);
|
$characterLimit = max(250, (int) $this->getValue('wordsToCachePerPage') * 6);
|
||||||
$contentSource = method_exists($page, 'contentRaw') ? $page->contentRaw() : $page->content();
|
|
||||||
$content = Text::truncate($this->plainText($contentSource), $characterLimit, '');
|
|
||||||
|
|
||||||
// Include the page to the cache
|
|
||||||
$cache[$pageKey] = array(
|
$cache[$pageKey] = array(
|
||||||
'title' => $this->plainText($page->title()),
|
'title' => $this->plainText($page->title()),
|
||||||
'description' => $this->plainText($page->description()),
|
'description' => $this->plainText($page->description()),
|
||||||
'content' => $content,
|
'content' => Text::truncate($this->plainText($this->pageRawContent($page)), $characterLimit, ''),
|
||||||
'tags' => '',
|
'tags' => '',
|
||||||
'category' => ''
|
'category' => ''
|
||||||
);
|
);
|
||||||
|
|
||||||
// Add tags if enabled
|
|
||||||
if ($this->getValue('searchInTags')) {
|
if ($this->getValue('searchInTags')) {
|
||||||
$pageTags = $page->tags(true);
|
$pageTags = $page->tags(true);
|
||||||
if (!empty($pageTags)) {
|
if (!empty($pageTags)) {
|
||||||
$cache[$pageKey]['tags'] = implode(' ', $pageTags);
|
$cache[$pageKey]['tags'] = $this->plainText(implode(' ', $pageTags));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add category if enabled
|
|
||||||
if ($this->getValue('searchInCategories')) {
|
if ($this->getValue('searchInCategories')) {
|
||||||
$categoryKey = $page->categoryKey();
|
$cache[$pageKey]['category'] = $this->plainText($page->category());
|
||||||
if (!empty($categoryKey) && isset($categories)) {
|
|
||||||
try {
|
|
||||||
$category = new Category($categoryKey);
|
|
||||||
$cache[$pageKey]['category'] = $this->plainText($category->name());
|
|
||||||
} catch (Exception $e) {
|
|
||||||
// Category not found
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate JSON file with the cache
|
$workspace = $this->workspace();
|
||||||
|
if (!is_dir($workspace)) {
|
||||||
|
$permissions = defined('DIR_PERMISSIONS') ? DIR_PERMISSIONS : 0755;
|
||||||
|
mkdir($workspace, $permissions, true);
|
||||||
|
}
|
||||||
|
|
||||||
$json = json_encode($cache, JSON_UNESCAPED_UNICODE);
|
$json = json_encode($cache, JSON_UNESCAPED_UNICODE);
|
||||||
return file_put_contents($this->cacheFile(), $json, LOCK_EX);
|
return file_put_contents($this->cacheFile(), $json, LOCK_EX);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns the absolute path of the cache file
|
|
||||||
private function cacheFile()
|
private function cacheFile()
|
||||||
{
|
{
|
||||||
return $this->workspace() . 'cache.json';
|
return $this->workspace() . 'cache.json';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search text inside the cache
|
|
||||||
// Returns an array with the pages keys related to the text
|
|
||||||
// The array is sorted by score
|
|
||||||
private function search($text)
|
private function search($text)
|
||||||
{
|
{
|
||||||
$text = trim($text);
|
$query = $this->normalizeQuery($text);
|
||||||
|
if (mb_strlen($query, CHARSET) < $this->minChars()) {
|
||||||
// Check minimum characters
|
|
||||||
$minChars = max(1, (int) $this->getValue('minChars'));
|
|
||||||
if (mb_strlen($text, 'UTF-8') < $minChars) {
|
|
||||||
return array();
|
return array();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read the cache file
|
$cache = $this->loadCache();
|
||||||
$cacheFile = $this->cacheFile();
|
$terms = $this->queryTerms($query);
|
||||||
if (!file_exists($cacheFile)) {
|
if (empty($cache) || empty($terms)) {
|
||||||
$this->createCache();
|
|
||||||
}
|
|
||||||
|
|
||||||
$json = file_get_contents($cacheFile);
|
|
||||||
$cache = json_decode($json, true);
|
|
||||||
|
|
||||||
if (!is_array($cache)) {
|
|
||||||
$this->createCache();
|
|
||||||
$json = file_get_contents($cacheFile);
|
|
||||||
$cache = json_decode($json, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (empty($cache) || !is_array($cache)) {
|
|
||||||
return array();
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($this->cacheContainsStoredSearchFields($cache)) {
|
|
||||||
$this->createCache();
|
|
||||||
$json = file_get_contents($cacheFile);
|
|
||||||
$cache = json_decode($json, true);
|
|
||||||
if (empty($cache) || !is_array($cache)) {
|
|
||||||
return array();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$terms = $this->tokenizeSearchTerm($text);
|
|
||||||
if (empty($terms)) {
|
|
||||||
return array();
|
return array();
|
||||||
}
|
}
|
||||||
|
|
||||||
$scores = array();
|
$scores = array();
|
||||||
foreach ($cache as $pageKey => $data) {
|
foreach ($cache as $pageKey => $pageData) {
|
||||||
if (!is_array($data)) {
|
$score = $this->scorePage($pageData, $query, $terms);
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
$score = $this->scoreSearchResult($data, $text, $terms);
|
|
||||||
if ($score > 0) {
|
if ($score > 0) {
|
||||||
$scores[$pageKey] = $score;
|
$scores[$pageKey] = $score;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (empty($scores)) {
|
||||||
|
return array();
|
||||||
|
}
|
||||||
|
|
||||||
arsort($scores, SORT_NUMERIC);
|
arsort($scores, SORT_NUMERIC);
|
||||||
return array_keys($scores);
|
return array_keys($scores);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function cacheContainsStoredSearchFields($cache)
|
private function loadCache()
|
||||||
{
|
{
|
||||||
foreach ($cache as $data) {
|
$cacheFile = $this->cacheFile();
|
||||||
return is_array($data) && isset($data['search']);
|
if (!file_exists($cacheFile)) {
|
||||||
|
$this->createCache();
|
||||||
|
}
|
||||||
|
|
||||||
|
$cache = $this->readCache();
|
||||||
|
if (!is_array($cache) || $this->cacheIsLegacy($cache)) {
|
||||||
|
$this->createCache();
|
||||||
|
$cache = $this->readCache();
|
||||||
|
}
|
||||||
|
|
||||||
|
return is_array($cache) ? $cache : array();
|
||||||
|
}
|
||||||
|
|
||||||
|
private function readCache()
|
||||||
|
{
|
||||||
|
$cacheFile = $this->cacheFile();
|
||||||
|
if (!file_exists($cacheFile)) {
|
||||||
|
return array();
|
||||||
|
}
|
||||||
|
|
||||||
|
$json = file_get_contents($cacheFile);
|
||||||
|
if ($json === false || $json === '') {
|
||||||
|
return array();
|
||||||
|
}
|
||||||
|
|
||||||
|
$cache = json_decode($json, true);
|
||||||
|
return is_array($cache) ? $cache : array();
|
||||||
|
}
|
||||||
|
|
||||||
|
private function cacheIsLegacy($cache)
|
||||||
|
{
|
||||||
|
foreach ($cache as $pageData) {
|
||||||
|
if (is_array($pageData) && isset($pageData['search'])) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function buildSearchFields($data)
|
private function scorePage($pageData, $query, $terms)
|
||||||
{
|
{
|
||||||
$fields = array(
|
if (!is_array($pageData)) {
|
||||||
'title' => $this->normalizeSearchText(isset($data['title']) ? $data['title'] : ''),
|
return 0;
|
||||||
'description' => $this->normalizeSearchText(isset($data['description']) ? $data['description'] : ''),
|
|
||||||
'content' => $this->normalizeSearchText(isset($data['content']) ? $data['content'] : ''),
|
|
||||||
'tags' => $this->normalizeSearchText(isset($data['tags']) ? $data['tags'] : ''),
|
|
||||||
'category' => $this->normalizeSearchText(isset($data['category']) ? $data['category'] : '')
|
|
||||||
);
|
|
||||||
$fields['all'] = trim(implode(' ', $fields));
|
|
||||||
|
|
||||||
return $fields;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function searchableFields($data)
|
|
||||||
{
|
|
||||||
if (isset($data['search']) && is_array($data['search'])) {
|
|
||||||
return $data['search'] + array(
|
|
||||||
'title' => '',
|
|
||||||
'description' => '',
|
|
||||||
'content' => '',
|
|
||||||
'tags' => '',
|
|
||||||
'category' => '',
|
|
||||||
'all' => ''
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return $this->buildSearchFields($data);
|
$fields = array(
|
||||||
|
'title' => $this->normalizeSearchText($this->arrayValue($pageData, 'title')),
|
||||||
|
'tags' => $this->normalizeSearchText($this->arrayValue($pageData, 'tags')),
|
||||||
|
'category' => $this->normalizeSearchText($this->arrayValue($pageData, 'category')),
|
||||||
|
'description' => $this->normalizeSearchText($this->arrayValue($pageData, 'description')),
|
||||||
|
'content' => $this->normalizeSearchText($this->arrayValue($pageData, 'content'))
|
||||||
|
);
|
||||||
|
|
||||||
|
$weights = array(
|
||||||
|
'title' => 100,
|
||||||
|
'tags' => 80,
|
||||||
|
'category' => 70,
|
||||||
|
'description' => 45,
|
||||||
|
'content' => 20
|
||||||
|
);
|
||||||
|
$score = 0;
|
||||||
|
$matches = 0;
|
||||||
|
|
||||||
|
foreach ($fields as $fieldName => $fieldText) {
|
||||||
|
if ($fieldText === '') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (strpos($fieldText, $query) !== false) {
|
||||||
|
$score += $weights[$fieldName] * 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach ($terms as $term) {
|
||||||
|
$termScore = $this->scoreTerm($fieldText, $term, $weights[$fieldName]);
|
||||||
|
if ($termScore > 0) {
|
||||||
|
$score += $termScore;
|
||||||
|
$matches++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($matches === 0) {
|
||||||
|
$score += $this->fuzzyTitleScore($fields, $terms);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (count($terms) > 1 && $matches >= count($terms)) {
|
||||||
|
$score += 40;
|
||||||
|
}
|
||||||
|
|
||||||
|
return $score;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function scoreTerm($fieldText, $term, $weight)
|
||||||
|
{
|
||||||
|
if ($fieldText === '' || $term === '') {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
$quoted = preg_quote($term, '/');
|
||||||
|
$wholeWordMatches = preg_match_all('/(^| )' . $quoted . '(?= |$)/u', $fieldText, $matches);
|
||||||
|
if ($wholeWordMatches > 0) {
|
||||||
|
return $weight + min(50, $wholeWordMatches * 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (mb_strlen($term, CHARSET) >= 4 && strpos($fieldText, $term) !== false) {
|
||||||
|
return (int) ($weight * 0.45);
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function fuzzyTitleScore($fields, $terms)
|
||||||
|
{
|
||||||
|
$text = trim($fields['title'] . ' ' . $fields['tags'] . ' ' . $fields['category']);
|
||||||
|
if ($text === '') {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
$words = $this->splitWords($text);
|
||||||
|
if (empty($words)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
$score = 0;
|
||||||
|
foreach ($terms as $term) {
|
||||||
|
if (mb_strlen($term, CHARSET) < 4) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
$allowedDistance = (mb_strlen($term, CHARSET) <= 5) ? 1 : 2;
|
||||||
|
foreach ($words as $word) {
|
||||||
|
if (abs(strlen($word) - strlen($term)) > $allowedDistance) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (levenshtein($term, $word) <= $allowedDistance) {
|
||||||
|
$score += 30;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return $score;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function normalizeQuery($text)
|
||||||
|
{
|
||||||
|
$text = $this->plainText(urldecode((string) $text));
|
||||||
|
if (mb_strlen($text, CHARSET) > 120) {
|
||||||
|
$text = mb_substr($text, 0, 120, CHARSET);
|
||||||
|
}
|
||||||
|
|
||||||
|
return $text;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function searchTermFromUrl()
|
||||||
|
{
|
||||||
|
$term = $this->webhook('search', true, false);
|
||||||
|
if (!is_string($term)) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
return $this->normalizeQuery(trim($term, '/'));
|
||||||
|
}
|
||||||
|
|
||||||
|
private function queryTerms($query)
|
||||||
|
{
|
||||||
|
$normalized = $this->normalizeSearchText($query);
|
||||||
|
$terms = array();
|
||||||
|
foreach ($this->splitWords($normalized) as $word) {
|
||||||
|
if (mb_strlen($word, CHARSET) >= 2) {
|
||||||
|
$terms[$word] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return array_keys($terms);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function splitWords($text)
|
||||||
|
{
|
||||||
|
$words = preg_split('/\s+/u', trim((string) $text), -1, PREG_SPLIT_NO_EMPTY);
|
||||||
|
if (!is_array($words)) {
|
||||||
|
$words = preg_split('/\s+/', trim((string) $text), -1, PREG_SPLIT_NO_EMPTY);
|
||||||
|
}
|
||||||
|
|
||||||
|
return is_array($words) ? $words : array();
|
||||||
}
|
}
|
||||||
|
|
||||||
private function normalizeSearchText($text)
|
private function normalizeSearchText($text)
|
||||||
{
|
{
|
||||||
$text = $this->plainText($text);
|
$text = $this->plainText($text);
|
||||||
|
$originalText = $text;
|
||||||
if (function_exists('transliterator_transliterate')) {
|
if (function_exists('transliterator_transliterate')) {
|
||||||
$transliterated = @transliterator_transliterate('Any-Latin; Latin-ASCII;', $text);
|
$transliterated = @transliterator_transliterate('Any-Latin; Latin-ASCII;', $text);
|
||||||
if ($transliterated !== false) {
|
if ($transliterated !== false) {
|
||||||
$text = $transliterated;
|
$text = $transliterated;
|
||||||
}
|
}
|
||||||
} elseif (function_exists('iconv')) {
|
} elseif (function_exists('iconv')) {
|
||||||
$transliterated = @iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $text);
|
$transliterated = @iconv(CHARSET, 'ASCII//TRANSLIT//IGNORE', $text);
|
||||||
if ($transliterated !== false) {
|
if ($transliterated !== false) {
|
||||||
$text = $transliterated;
|
$text = $transliterated;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$text = mb_strtolower($text, 'UTF-8');
|
$text = mb_strtolower($text, CHARSET);
|
||||||
$normalized = preg_replace('/[^\p{L}\p{N}]+/u', ' ', $text);
|
$text = preg_replace('/[^\p{L}\p{N}]+/u', ' ', $text);
|
||||||
if ($normalized === null) {
|
if ($text === null) {
|
||||||
$normalized = preg_replace('/[^a-z0-9]+/i', ' ', $text);
|
$text = preg_replace('/[^a-z0-9]+/i', ' ', $originalText);
|
||||||
}
|
}
|
||||||
if ($normalized === null) {
|
if ($text === null) {
|
||||||
$normalized = $text;
|
return '';
|
||||||
}
|
|
||||||
$collapsed = preg_replace('/\s+/u', ' ', $normalized);
|
|
||||||
if ($collapsed === null) {
|
|
||||||
$collapsed = preg_replace('/\s+/', ' ', $normalized);
|
|
||||||
}
|
|
||||||
if ($collapsed === null) {
|
|
||||||
$collapsed = $normalized;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return trim((string) $collapsed);
|
$text = preg_replace('/\s+/u', ' ', $text);
|
||||||
|
return trim((string) $text);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function tokenizeSearchTerm($text)
|
private function pageRawContent($page)
|
||||||
{
|
{
|
||||||
$normalized = $this->normalizeSearchText($text);
|
if (method_exists($page, 'contentRaw')) {
|
||||||
$words = $this->splitWords($normalized);
|
return $page->contentRaw();
|
||||||
|
|
||||||
$tokens = array();
|
|
||||||
foreach ($words as $word) {
|
|
||||||
if (mb_strlen($word, 'UTF-8') >= 2) {
|
|
||||||
$tokens[$word] = true;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return array_keys($tokens);
|
return $page->content();
|
||||||
}
|
}
|
||||||
|
|
||||||
private function scoreSearchResult($data, $query, $terms)
|
private function minChars()
|
||||||
{
|
{
|
||||||
$fields = $this->searchableFields($data);
|
return max(1, (int) $this->getValue('minChars'));
|
||||||
$query = $this->normalizeSearchText($query);
|
|
||||||
$score = 0;
|
|
||||||
|
|
||||||
if (!empty($query)) {
|
|
||||||
$phraseWeights = array(
|
|
||||||
'title' => 220,
|
|
||||||
'tags' => 180,
|
|
||||||
'category' => 160,
|
|
||||||
'description' => 110,
|
|
||||||
'content' => 70
|
|
||||||
);
|
|
||||||
|
|
||||||
foreach ($phraseWeights as $field => $weight) {
|
|
||||||
if (!empty($fields[$field]) && strpos($fields[$field], $query) !== false) {
|
|
||||||
$score += $weight + min(60, $this->countOccurrences($fields[$field], $query) * 8);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$matchedTerms = 0;
|
|
||||||
foreach ($terms as $term) {
|
|
||||||
$termScore = 0;
|
|
||||||
$termScore += $this->scoreTermInField($fields['title'], $term, 95);
|
|
||||||
$termScore += $this->scoreTermInField($fields['tags'], $term, 80);
|
|
||||||
$termScore += $this->scoreTermInField($fields['category'], $term, 75);
|
|
||||||
$termScore += $this->scoreTermInField($fields['description'], $term, 45);
|
|
||||||
$termScore += $this->scoreTermInField($fields['content'], $term, 25);
|
|
||||||
|
|
||||||
if ($termScore === 0) {
|
|
||||||
$termScore += $this->fuzzyTermScore($term, trim($fields['title'] . ' ' . $fields['tags'] . ' ' . $fields['category'] . ' ' . $fields['description']), 24);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($termScore > 0) {
|
|
||||||
$matchedTerms++;
|
|
||||||
$score += $termScore;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($matchedTerms === 0) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (count($terms) > 1) {
|
|
||||||
if ($matchedTerms === count($terms)) {
|
|
||||||
$score += 45;
|
|
||||||
} else {
|
|
||||||
$score -= (count($terms) - $matchedTerms) * 25;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return max(0, $score);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private function scoreTermInField($field, $term, $weight)
|
private function isSearchRequest()
|
||||||
{
|
{
|
||||||
if (empty($field)) {
|
return $this->webhook('search', false, false);
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
$quoted = preg_quote($term, '/');
|
|
||||||
$fullMatches = preg_match_all('/(^| )' . $quoted . '(?= |$)/u', $field, $matches);
|
|
||||||
if ($fullMatches > 0) {
|
|
||||||
return $weight + min(40, $fullMatches * 6);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (mb_strlen($term, 'UTF-8') >= 4 && strpos($field, $term) !== false) {
|
|
||||||
return (int) ($weight * 0.55) + min(25, $this->countOccurrences($field, $term) * 4);
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private function countOccurrences($field, $needle)
|
private function arrayValue($array, $key)
|
||||||
{
|
{
|
||||||
if ($needle === '') {
|
return (is_array($array) && isset($array[$key])) ? $array[$key] : '';
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
return substr_count($field, $needle);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private function fuzzyTermScore($term, $field, $weight)
|
private function logSearchError($message)
|
||||||
{
|
{
|
||||||
if (mb_strlen($term, 'UTF-8') < 4 || empty($field)) {
|
if (class_exists('Log') && defined('LOG_SEP')) {
|
||||||
return 0;
|
Log::set(__METHOD__ . LOG_SEP . $message);
|
||||||
}
|
}
|
||||||
|
|
||||||
$words = $this->splitWords($field);
|
|
||||||
|
|
||||||
$bestScore = 0;
|
|
||||||
$checked = array();
|
|
||||||
$checkedCount = 0;
|
|
||||||
$termLength = strlen($term);
|
|
||||||
$allowedDistance = ($termLength <= 5) ? 1 : 2;
|
|
||||||
|
|
||||||
foreach ($words as $word) {
|
|
||||||
if (isset($checked[$word])) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
$checked[$word] = true;
|
|
||||||
|
|
||||||
if (strlen($word) > 64 || abs(strlen($word) - $termLength) > $allowedDistance) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
$checkedCount++;
|
|
||||||
if ($checkedCount > 250) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
$distance = levenshtein($term, $word);
|
|
||||||
if ($distance <= $allowedDistance) {
|
|
||||||
$bestScore = max($bestScore, $weight - ($distance * 5));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $bestScore;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user