- The medicine database is now much bigger and has a real update path.
Added: Live source updater for RxNorm, DailyMed, and openFDA. Admin dashboard panel to refresh medicine sources. CLI updater for periodic jobs: [scripts/update_medicine_catalog.php](/Users/tyemeclifford/Documents/GH/workout/scripts/update_medicine_catalog.php) Update history table: medication_source_updates. Curl fallback for external fetches when PHP stream DNS fails. README instructions for periodic updates. I also ran the imports: Current medication catalog count: 6,002. Latest source imports:RxNorm: 3,875 inserted, 1,125 updated. openFDA: 912 inserted, 88 updated. DailyMed: 95 inserted. Periodic update command: /Users/tyemeclifford/frankenphp php-cli scripts/update_medicine_catalog.php 5000 all
This commit is contained in:
Ty Clifford
@@ -23,6 +23,7 @@ function app_db(): PDO
|
||||
$pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
|
||||
$pdo->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
|
||||
$pdo->exec('PRAGMA foreign_keys = ON');
|
||||
$pdo->exec('PRAGMA busy_timeout = 5000');
|
||||
|
||||
app_init_db($pdo);
|
||||
|
||||
@@ -315,6 +316,16 @@ CREATE TABLE IF NOT EXISTS research_studies (
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS medication_source_updates (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
imported_count INTEGER NOT NULL DEFAULT 0,
|
||||
updated_count INTEGER NOT NULL DEFAULT 0,
|
||||
message TEXT DEFAULT '',
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_workouts_user_date ON workouts(user_id, scheduled_on);
|
||||
CREATE INDEX IF NOT EXISTS idx_nutrition_logs_user_date ON nutrition_logs(user_id, logged_on);
|
||||
CREATE INDEX IF NOT EXISTS idx_supplement_logs_user_date ON supplement_logs(user_id, taken_on);
|
||||
@@ -328,6 +339,7 @@ CREATE INDEX IF NOT EXISTS idx_appointments_user_date ON appointments(user_id, a
|
||||
CREATE INDEX IF NOT EXISTS idx_immunization_records_user_date ON immunization_records(user_id, administered_on);
|
||||
CREATE INDEX IF NOT EXISTS idx_medical_terms_term ON medical_terms(term);
|
||||
CREATE INDEX IF NOT EXISTS idx_medication_catalog_generic ON medication_catalog(generic_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_medication_source_updates_created ON medication_source_updates(created_at);
|
||||
SQL);
|
||||
|
||||
migrate_users_table($pdo);
|
||||
@@ -611,6 +623,12 @@ function seed_expanded_medication_catalog(PDO $pdo): void
|
||||
{
|
||||
$created = app_now();
|
||||
$rows = expanded_medication_catalog_rows();
|
||||
$existingCount = (int) $pdo->query('SELECT COUNT(*) FROM medication_catalog')->fetchColumn();
|
||||
$needsSearchText = (int) $pdo->query("SELECT COUNT(*) FROM medication_catalog WHERE search_terms IS NULL OR search_terms = ''")->fetchColumn();
|
||||
$needsChemicalNames = (int) $pdo->query("SELECT COUNT(*) FROM medication_catalog WHERE chemical_name IS NULL OR chemical_name = ''")->fetchColumn();
|
||||
if ($existingCount >= count($rows) && $needsSearchText === 0 && $needsChemicalNames === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
$select = $pdo->prepare('SELECT id FROM medication_catalog WHERE lower(generic_name) = lower(?) LIMIT 1');
|
||||
$insert = $pdo->prepare(
|
||||
@@ -1705,6 +1723,312 @@ function library_search_payload(array $query): array
|
||||
];
|
||||
}
|
||||
|
||||
function http_json(string $url, int $timeout = 20): ?array
|
||||
{
|
||||
$context = stream_context_create([
|
||||
'http' => [
|
||||
'method' => 'GET',
|
||||
'timeout' => $timeout,
|
||||
'header' => "Accept: application/json\r\nUser-Agent: NeonMedicalTracker/1.0\r\n",
|
||||
],
|
||||
]);
|
||||
$body = @file_get_contents($url, false, $context);
|
||||
if (($body === false || $body === '') && function_exists('shell_exec')) {
|
||||
$curl = trim((string) shell_exec('command -v curl 2>/dev/null'));
|
||||
if ($curl === '' && is_executable('/usr/bin/curl')) {
|
||||
$curl = '/usr/bin/curl';
|
||||
}
|
||||
if ($curl !== '') {
|
||||
$command = escapeshellarg($curl)
|
||||
. ' -fsSL --max-time ' . (int) $timeout
|
||||
. ' -H ' . escapeshellarg('Accept: application/json')
|
||||
. ' -A ' . escapeshellarg('NeonMedicalTracker/1.0')
|
||||
. ' ' . escapeshellarg($url)
|
||||
. ' 2>/dev/null';
|
||||
$body = shell_exec($command);
|
||||
}
|
||||
}
|
||||
if (!is_string($body) || $body === '') {
|
||||
return null;
|
||||
}
|
||||
$json = json_decode($body, true);
|
||||
return is_array($json) ? $json : null;
|
||||
}
|
||||
|
||||
function medication_import_row(string $name, string $source, string $brandNames = '', string $className = 'Imported medicine concept'): array
|
||||
{
|
||||
$cleanName = preg_replace('/\s+/', ' ', trim($name));
|
||||
if ($cleanName === '') {
|
||||
return [];
|
||||
}
|
||||
|
||||
$sourceLabel = match ($source) {
|
||||
'RxNorm' => 'RxNorm / NLM',
|
||||
'DailyMed' => 'DailyMed / NLM',
|
||||
'openFDA' => 'openFDA / FDA',
|
||||
default => $source,
|
||||
};
|
||||
|
||||
return [
|
||||
'generic_name' => strtolower($cleanName),
|
||||
'active_ingredient' => $cleanName,
|
||||
'chemical_name' => $cleanName,
|
||||
'brand_names' => $brandNames,
|
||||
'generic_brands' => $cleanName,
|
||||
'class_name' => $className,
|
||||
'drug_family' => 'Imported reference',
|
||||
'common_use' => "Imported from {$sourceLabel}. Review official labels and clinician guidance before use.",
|
||||
'tracking_note' => 'Use this entry as a reference/search starting point; verify dose, indication, and safety details with official labeling or a clinician.',
|
||||
'rxnorm_url' => medication_source_url('https://mor.nlm.nih.gov/RxNav/search?searchBy=String&searchTerm=', $cleanName),
|
||||
'medlineplus_url' => 'https://medlineplus.gov/druginformation.html',
|
||||
'pubchem_url' => medication_source_url('https://pubchem.ncbi.nlm.nih.gov/#query=', $cleanName),
|
||||
'dailymed_url' => medication_source_url('https://dailymed.nlm.nih.gov/dailymed/search.cfm?query=', $cleanName),
|
||||
'pubmed_url' => medication_source_url('https://pubmed.ncbi.nlm.nih.gov/?term=', $cleanName . ' medication'),
|
||||
'clinicaltrials_url' => medication_source_url('https://clinicaltrials.gov/search?term=', $cleanName),
|
||||
'source_name' => $sourceLabel,
|
||||
'search_terms' => implode(' ', [$cleanName, $brandNames, $className, $sourceLabel]),
|
||||
];
|
||||
}
|
||||
|
||||
function fetch_rxnorm_medication_rows(int $limit): array
|
||||
{
|
||||
$rows = [];
|
||||
foreach (['IN', 'PIN'] as $tty) {
|
||||
$json = http_json('https://rxnav.nlm.nih.gov/REST/allconcepts.json?tty=' . $tty);
|
||||
$concepts = $json['minConceptGroup']['minConcept'] ?? [];
|
||||
if (!is_array($concepts)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($concepts as $concept) {
|
||||
if (count($rows) >= $limit) {
|
||||
break 2;
|
||||
}
|
||||
$name = (string) ($concept['name'] ?? '');
|
||||
$row = medication_import_row($name, 'RxNorm');
|
||||
if ($row) {
|
||||
$rows[] = $row;
|
||||
}
|
||||
}
|
||||
}
|
||||
return $rows;
|
||||
}
|
||||
|
||||
function fetch_openfda_medication_rows(int $limit): array
|
||||
{
|
||||
$url = 'https://api.fda.gov/drug/label.json?count=openfda.generic_name.exact&limit=' . $limit;
|
||||
$json = http_json($url);
|
||||
$results = $json['results'] ?? [];
|
||||
if (!is_array($results)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$rows = [];
|
||||
foreach ($results as $result) {
|
||||
if (count($rows) >= $limit) {
|
||||
break;
|
||||
}
|
||||
$term = (string) ($result['term'] ?? '');
|
||||
$row = medication_import_row($term, 'openFDA');
|
||||
if ($row) {
|
||||
$rows[] = $row;
|
||||
}
|
||||
}
|
||||
return $rows;
|
||||
}
|
||||
|
||||
function fetch_dailymed_medication_rows(int $limit): array
|
||||
{
|
||||
$url = 'https://dailymed.nlm.nih.gov/dailymed/services/v2/drugnames.json';
|
||||
$json = http_json($url);
|
||||
$data = $json['data'] ?? $json['drugNames'] ?? $json['results'] ?? [];
|
||||
if (!is_array($data)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$rows = [];
|
||||
foreach ($data as $item) {
|
||||
if (count($rows) >= $limit) {
|
||||
break;
|
||||
}
|
||||
$name = is_array($item)
|
||||
? (string) ($item['drug_name'] ?? $item['name'] ?? $item['term'] ?? '')
|
||||
: (string) $item;
|
||||
$row = medication_import_row($name, 'DailyMed');
|
||||
if ($row) {
|
||||
$rows[] = $row;
|
||||
}
|
||||
}
|
||||
return $rows;
|
||||
}
|
||||
|
||||
function merge_text_values(string $current, string $incoming): string
|
||||
{
|
||||
$parts = [];
|
||||
foreach (explode(',', $current . ',' . $incoming) as $part) {
|
||||
$value = trim($part);
|
||||
if ($value === '') {
|
||||
continue;
|
||||
}
|
||||
$parts[strtolower($value)] = $value;
|
||||
}
|
||||
return implode(', ', array_values($parts));
|
||||
}
|
||||
|
||||
function upsert_imported_medication_rows(PDO $pdo, array $rows): array
|
||||
{
|
||||
$created = app_now();
|
||||
$select = $pdo->prepare('SELECT * FROM medication_catalog WHERE lower(generic_name) = lower(?) LIMIT 1');
|
||||
$insert = $pdo->prepare(
|
||||
"INSERT INTO medication_catalog
|
||||
(generic_name, active_ingredient, chemical_name, brand_names, generic_brands, class_name, drug_family,
|
||||
common_use, tracking_note, rxnorm_url, medlineplus_url, pubchem_url, dailymed_url, pubmed_url,
|
||||
clinicaltrials_url, source_name, search_terms, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
||||
);
|
||||
$update = $pdo->prepare(
|
||||
"UPDATE medication_catalog
|
||||
SET active_ingredient = ?, chemical_name = ?, brand_names = ?, generic_brands = ?,
|
||||
class_name = ?, drug_family = ?, common_use = ?, tracking_note = ?, rxnorm_url = ?,
|
||||
medlineplus_url = ?, pubchem_url = ?, dailymed_url = ?, pubmed_url = ?,
|
||||
clinicaltrials_url = ?, source_name = ?, search_terms = ?
|
||||
WHERE id = ?"
|
||||
);
|
||||
|
||||
$seen = [];
|
||||
$inserted = 0;
|
||||
$updated = 0;
|
||||
foreach ($rows as $row) {
|
||||
if (!$row || empty($row['generic_name'])) {
|
||||
continue;
|
||||
}
|
||||
$key = strtolower((string) $row['generic_name']);
|
||||
if (isset($seen[$key])) {
|
||||
continue;
|
||||
}
|
||||
$seen[$key] = true;
|
||||
|
||||
$select->execute([$row['generic_name']]);
|
||||
$existing = $select->fetch();
|
||||
if (!$existing) {
|
||||
$insert->execute([
|
||||
$row['generic_name'],
|
||||
$row['active_ingredient'],
|
||||
$row['chemical_name'],
|
||||
$row['brand_names'],
|
||||
$row['generic_brands'],
|
||||
$row['class_name'],
|
||||
$row['drug_family'],
|
||||
$row['common_use'],
|
||||
$row['tracking_note'],
|
||||
$row['rxnorm_url'],
|
||||
$row['medlineplus_url'],
|
||||
$row['pubchem_url'],
|
||||
$row['dailymed_url'],
|
||||
$row['pubmed_url'],
|
||||
$row['clinicaltrials_url'],
|
||||
$row['source_name'],
|
||||
$row['search_terms'],
|
||||
$created,
|
||||
]);
|
||||
$inserted++;
|
||||
continue;
|
||||
}
|
||||
|
||||
$next = [];
|
||||
foreach (['active_ingredient', 'chemical_name', 'class_name', 'drug_family', 'common_use', 'tracking_note', 'rxnorm_url', 'medlineplus_url', 'pubchem_url', 'dailymed_url', 'pubmed_url', 'clinicaltrials_url'] as $field) {
|
||||
$next[$field] = trim((string) ($existing[$field] ?? '')) !== '' ? $existing[$field] : $row[$field];
|
||||
}
|
||||
$next['brand_names'] = merge_text_values((string) ($existing['brand_names'] ?? ''), (string) $row['brand_names']);
|
||||
$next['generic_brands'] = merge_text_values((string) ($existing['generic_brands'] ?? ''), (string) $row['generic_brands']);
|
||||
$next['source_name'] = merge_text_values((string) ($existing['source_name'] ?? ''), (string) $row['source_name']);
|
||||
$next['search_terms'] = trim((string) ($existing['search_terms'] ?? '') . ' ' . (string) $row['search_terms']);
|
||||
|
||||
$update->execute([
|
||||
$next['active_ingredient'],
|
||||
$next['chemical_name'],
|
||||
$next['brand_names'],
|
||||
$next['generic_brands'],
|
||||
$next['class_name'],
|
||||
$next['drug_family'],
|
||||
$next['common_use'],
|
||||
$next['tracking_note'],
|
||||
$next['rxnorm_url'],
|
||||
$next['medlineplus_url'],
|
||||
$next['pubchem_url'],
|
||||
$next['dailymed_url'],
|
||||
$next['pubmed_url'],
|
||||
$next['clinicaltrials_url'],
|
||||
$next['source_name'],
|
||||
$next['search_terms'],
|
||||
(int) $existing['id'],
|
||||
]);
|
||||
$updated++;
|
||||
}
|
||||
|
||||
return ['inserted' => $inserted, 'updated' => $updated, 'seen' => count($seen)];
|
||||
}
|
||||
|
||||
function record_medication_source_update(PDO $pdo, string $source, string $status, int $inserted, int $updated, string $message): void
|
||||
{
|
||||
$stmt = $pdo->prepare(
|
||||
"INSERT INTO medication_source_updates
|
||||
(source, status, imported_count, updated_count, message, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)"
|
||||
);
|
||||
$stmt->execute([$source, $status, $inserted, $updated, $message, app_now()]);
|
||||
}
|
||||
|
||||
function refresh_medication_sources(array $data): array
|
||||
{
|
||||
$pdo = app_db();
|
||||
if (array_key_exists('acting_user_id', $data) || array_key_exists('user_id', $data)) {
|
||||
require_admin($pdo, $data);
|
||||
}
|
||||
|
||||
$limit = max(25, min(5000, int_value($data, 'limit', 1000)));
|
||||
$sources = [
|
||||
'RxNorm' => 'fetch_rxnorm_medication_rows',
|
||||
'DailyMed' => 'fetch_dailymed_medication_rows',
|
||||
'openFDA' => 'fetch_openfda_medication_rows',
|
||||
];
|
||||
$requestedSource = text_value($data, 'source', 'all');
|
||||
if ($requestedSource !== 'all') {
|
||||
$sources = array_intersect_key($sources, [$requestedSource => true]);
|
||||
}
|
||||
|
||||
$summary = [];
|
||||
$totalInserted = 0;
|
||||
$totalUpdated = 0;
|
||||
foreach ($sources as $source => $fetcher) {
|
||||
try {
|
||||
$rows = $fetcher($limit);
|
||||
if (!$rows) {
|
||||
record_medication_source_update($pdo, $source, 'empty', 0, 0, 'No rows returned. The source may be unavailable or rate-limited.');
|
||||
$summary[] = ['source' => $source, 'status' => 'empty', 'inserted' => 0, 'updated' => 0, 'message' => 'No rows returned.'];
|
||||
continue;
|
||||
}
|
||||
$result = upsert_imported_medication_rows($pdo, $rows);
|
||||
$totalInserted += $result['inserted'];
|
||||
$totalUpdated += $result['updated'];
|
||||
$message = "Fetched {$result['seen']} unique rows.";
|
||||
record_medication_source_update($pdo, $source, 'ok', $result['inserted'], $result['updated'], $message);
|
||||
$summary[] = ['source' => $source, 'status' => 'ok', 'inserted' => $result['inserted'], 'updated' => $result['updated'], 'message' => $message];
|
||||
} catch (Throwable $exception) {
|
||||
record_medication_source_update($pdo, $source, 'error', 0, 0, $exception->getMessage());
|
||||
$summary[] = ['source' => $source, 'status' => 'error', 'inserted' => 0, 'updated' => 0, 'message' => $exception->getMessage()];
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
'inserted' => $totalInserted,
|
||||
'updated' => $totalUpdated,
|
||||
'catalog_count' => (int) db_value($pdo, 'SELECT COUNT(*) FROM medication_catalog'),
|
||||
'sources' => $summary,
|
||||
'recent_updates' => db_all($pdo, 'SELECT * FROM medication_source_updates ORDER BY created_at DESC, id DESC LIMIT 10'),
|
||||
];
|
||||
}
|
||||
|
||||
function create_user(array $data): array
|
||||
{
|
||||
$pdo = app_db();
|
||||
|
||||
Reference in New Issue
Block a user