From 2c5d6762a99869672fe3865680e104c4ee743371 Mon Sep 17 00:00:00 2001 From: lion <120344285@qq.com> Date: Tue, 28 Jul 2026 15:57:29 +0800 Subject: [PATCH] =?UTF-8?q?=E8=80=81=E5=B8=88=E5=BA=93=E7=A0=94=E7=A9=B6?= =?UTF-8?q?=E6=96=B9=E5=90=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Controllers/Admin/CrawlJobController.php | 37 +- .../Crawl/Adapters/FacultyListHtmlAdapter.php | 494 +++++++++++++++++- app/Services/Crawl/CrawlImportService.php | 269 +++++++--- app/Services/Crawl/CrawlJobRunnerService.php | 34 +- config/crawl.php | 6 +- .../Unit/CrawlImportTeacherFillEmptyTest.php | 76 +++ tests/Unit/FacultyListHtmlAdapterTest.php | 186 ++++++- 7 files changed, 958 insertions(+), 144 deletions(-) create mode 100644 tests/Unit/CrawlImportTeacherFillEmptyTest.php diff --git a/app/Http/Controllers/Admin/CrawlJobController.php b/app/Http/Controllers/Admin/CrawlJobController.php index e1a7f12..bdc8c5e 100644 --- a/app/Http/Controllers/Admin/CrawlJobController.php +++ b/app/Http/Controllers/Admin/CrawlJobController.php @@ -227,6 +227,8 @@ class CrawlJobController extends Controller 'imported_primary' => $result['imported_primary'], 'skipped' => $result['skipped'], 'failed' => $result['failed'], + 'teachers_imported' => $result['teachers_imported'] ?? 0, + 'teachers_updated' => $result['teachers_updated'] ?? 0, 'items_imported' => $job->fresh()->items_imported, ], $this->buildImportSuccessMessage($job, $result)); } @@ -432,7 +434,9 @@ class CrawlJobController extends Controller } if ($job->target_type === 'teacher') { - $imported = (int) ($importResult['teachers_imported'] ?? 0); + $created = (int) ($importResult['teachers_imported'] ?? 0); + $updated = (int) ($importResult['teachers_updated'] ?? 0); + $imported = $created + $updated; $duplicateCount = $this->countTeacherDuplicateItems($job); $summary = sprintf( '已从 %s 抓取 %d 位老师，已入库 %d 位老师', @@ -440,8 +444,18 @@ class CrawlJobController extends Controller $fetched, $imported, ); + if ($created > 0 || $updated > 0) { + $parts = []; + if ($created > 0) { + $parts[] = sprintf('新建 %d', $created); + } + if ($updated > 0) { + $parts[] = sprintf('补全 %d', $updated); + } + $summary .= '（'.implode('，', $parts).'）'; + } if ($duplicateCount > 0) { - $summary .= sprintf('，跳过 %d 位（老师库中已存在）', $duplicateCount); + $summary .= sprintf('，跳过 %d 位（已存在且无需补全）', $duplicateCount); } $failedCount = (int) ($importResult['failed'] ?? 0); if ($failedCount > 0) { @@ -508,13 +522,26 @@ class CrawlJobController extends Controller } if ($job->target_type === 'teacher') { - $teachers = (int) ($importResult['teachers_imported'] ?? 0); + $created = (int) ($importResult['teachers_imported'] ?? 0); + $updated = (int) ($importResult['teachers_updated'] ?? 0); + $teachers = $created + $updated; $duplicateCount = $this->countTeacherDuplicateItems($job); + $message = "抓取完成，已入库 {$teachers} 位老师"; + if ($created > 0 || $updated > 0) { + $parts = []; + if ($created > 0) { + $parts[] = "新建 {$created}"; + } + if ($updated > 0) { + $parts[] = "补全 {$updated}"; + } + $message .= '（'.implode('，', $parts).'）'; + } if ($duplicateCount > 0) { - return "抓取完成，已入库 {$teachers} 位老师，跳过 {$duplicateCount} 位"; + $message .= "，跳过 {$duplicateCount} 位"; } - return "抓取完成，已入库 {$teachers} 位老师"; + return $message; } $news = (int) ($importResult['news_imported'] ?? 0); diff --git a/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php b/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php index 01498a7..ac4f1bd 100644 --- a/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php +++ b/app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php @@ -183,7 +183,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface if ($enrichBudget <= 0) { break; } - if ($this->itemHasEmail($item) || ! $item->canonicalUrl) { + if (! $this->itemNeedsProfileEnrich($item)) { continue; } $fetchMap[$index] = $item; @@ -215,9 +215,11 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface foreach ($batchPending as $index => $item) { $body = $this->responseBodyFromPoolResult($responses[(string) $index] ?? null); if ($body !== null) { - $email = $this->extractEmailFromProfileHtml($body); - if ($email) { - $item = $this->applyEmailToItem($item, $email); + if (! $this->itemHasEmail($item)) { + $email = $this->extractEmailFromProfileHtml($body); + if ($email) { + $item = $this->applyEmailToItem($item, $email); + } } $item = $this->applyProfileMetadataToItem($item, $body); } @@ -229,7 +231,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface foreach ($items as $index => $item) { if (isset($fetchedBodies[$index])) { $result[] = $fetchedBodies[$index]; - } elseif (! $this->itemHasEmail($item) && $item->canonicalUrl) { + } elseif ($this->itemNeedsProfileEnrich($item)) { $result[] = $this->markItemProfileEnrichSkipped($item); } else { $result[] = $item; @@ -239,6 +241,62 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface return $result; } + protected function itemNeedsProfileEnrich(CrawlItemDto $item): bool + { + if (! $item->canonicalUrl) { + return false; + } + + return ! $this->itemHasEmail($item) + || ! $this->itemHasPhone($item) + || ! $this->itemHasResearchDirections($item) + || ! $this->itemHasBio($item); + } + + protected function itemHasPhone(CrawlItemDto $item): bool + { + $lead = $item->extra['lead_author'] ?? null; + $phone = is_array($lead) ? trim((string) ($lead['phone'] ?? '')) : ''; + if ($phone === '') { + $phone = trim((string) ($item->extra['phone'] ?? '')); + } + + return $phone !== ''; + } + + protected function itemHasResearchDirections(CrawlItemDto $item): bool + { + $names = $item->extra['research_direction_names'] ?? null; + if (is_array($names) && $names !== []) { + return true; + } + + $lead = $item->extra['lead_author'] ?? null; + if (is_array($lead)) { + $leadNames = $lead['research_direction_names'] ?? null; + if (is_array($leadNames) && $leadNames !== []) { + return true; + } + } + + return false; + } + + protected function itemHasBio(CrawlItemDto $item): bool + { + $bio = trim((string) ($item->extra['bio'] ?? '')); + if ($bio !== '') { + return true; + } + + $lead = $item->extra['lead_author'] ?? null; + if (is_array($lead) && trim((string) ($lead['bio'] ?? '')) !== '') { + return true; + } + + return false; + } + /** * @param array $params */ @@ -248,9 +306,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface return 0; } - $configured = (int) ($params['profile_enrich_max'] ?? config('crawl.faculty.profile_enrich_max', 32)); + $configured = (int) ($params['profile_enrich_max'] ?? config('crawl.faculty.profile_enrich_max', 200)); - return max(0, min($itemCount, min(200, $configured))); + return max(0, min($itemCount, min(500, $configured))); } /** @@ -264,7 +322,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface protected function markItemProfileEnrichSkipped(CrawlItemDto $item): CrawlItemDto { - if ($this->itemHasEmail($item)) { + if (! $this->itemNeedsProfileEnrich($item)) { return $item; } @@ -349,17 +407,22 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface protected function extractEmailFromProfileHtml(string $html): ?string { + $scoped = $this->profileContentHtml($html); + $labeledPatterns = [ - '/电子邮箱[：:]\s*<\/strong>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/电子邮箱[：:]\s*<\/(?:strong|span|b)>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', '/电子邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', '/电子信箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', - '/E-?mail[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/iu', - '/邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', '/电子邮件[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/E-?mail[：:]\s*(?:<[^>]+>\s*)*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/iu', + '/details-tag">\s*电子邮件\s*<\/span>\s*\s*(?:]*>)?\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/电子邮箱[：:]\s*<\/span>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u', + '/mailto:([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i', ]; foreach ($labeledPatterns as $pattern) { - if (preg_match($pattern, $html, $match)) { + if (preg_match($pattern, $scoped, $match)) { $email = CrawlAuthorParser::normalizeEmail($match[1]); if ($email && ! $this->isNoiseEmail($email)) { return $email; @@ -370,7 +433,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface $candidates = []; if (preg_match_all( '#([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})#', - $html, + $scoped, $emailMatches, )) { foreach ($emailMatches[1] as $raw) { @@ -398,10 +461,16 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface protected function isNoiseEmail(string $email): bool { - return (bool) preg_match( - '/^(noreply|no-reply|admin|webmaster|postmaster|root|support|service|info|contact)@/i', - $email, - ); + $local = strtolower((string) strstr($email, '@', true)); + + if (preg_match( + '/^(noreply|no-reply|admin|webmaster|postmaster|root|support|service|info|contact|see|scs|soai|icisee|sais|smse|faculty|office|dean)$/i', + $local, + )) { + return true; + } + + return false; } protected function fetchHtml(string $url): string @@ -501,6 +570,11 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface return $items; } + $items = $this->extractFromSoaiFacultyList($html, $keywords, $sourceUrl); + if ($items !== []) { + return $items; + } + $items = $this->extractFromVsbFacultyTable($html, $keywords, $sourceUrl); if ($items !== []) { return $items; @@ -519,6 +593,76 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface return $this->extractFromStaffPanelList($html, $keywords, $sourceUrl); } + /** + * 交大人工智能学院：/cn/facultydetails/... 卡片列表。 + * + * @param list $keywords + * @return list + */ + protected function extractFromSoaiFacultyList(string $html, array $keywords, string $sourceUrl): array + { + if (! preg_match_all( + '#]*?/cn/facultydetails/[^\'"]+[^>]*)>([^<]+)#u', + $html, + $matches, + PREG_SET_ORDER, + )) { + return []; + } + + $pageUniversity = $this->inferUniversityFromSource($sourceUrl, $html); + $defaultCollege = $this->inferCollegeFromPageTitle($html) ?? '人工智能学院'; + $items = []; + $seen = []; + + foreach ($matches as $match) { + $attrs = (string) $match[1]; + $name = CrawlAuthorParser::cleanText($match[2]) ?? ''; + if ($name === '' || ! $this->looksLikePersonName($name)) { + continue; + } + if (! preg_match('#\bhref=[\'"]([^\'"]+)[\'"]#u', $attrs, $hrefMatch)) { + continue; + } + + $href = html_entity_decode($hrefMatch[1], ENT_QUOTES | ENT_HTML5, 'UTF-8'); + $profileUrl = $this->resolveUrl($href, $sourceUrl); + $dedupeKey = $profileUrl ?: ('name:'.md5($name)); + if (isset($seen[$dedupeKey])) { + continue; + } + + $plain = trim($name.' '.($defaultCollege ?? '')); + if (! $this->matchesKeywords($plain, $keywords)) { + continue; + } + + $academicTitle = null; + $windowStart = max(0, (int) strpos($html, $match[0]) - 50); + $window = substr($html, $windowStart, 500); + if (preg_match('#职称[：:]\s*([^<]+)#u', $window, $titleMatch)) { + $academicTitle = CrawlAuthorParser::cleanText($titleMatch[1]); + } + + $seen[$dedupeKey] = true; + $items[] = $this->makeFacultyItem( + externalKey: 'faculty:'.md5($dedupeKey), + name: $name, + profileUrl: $profileUrl, + email: null, + affiliation: $defaultCollege, + universityName: $pageUniversity ?? '上海交通大学', + summary: $defaultCollege ? '单位：'.$defaultCollege : null, + keywords: $keywords, + academicTitle: $academicTitle, + platform: 'faculty_html_soai', + bio: null, + ); + } + + return $items; + } + /** * 南大 Sudy CMS：ul.news_list 内 news_title / news_title1 链接（frontier、ic 等）。 * @@ -656,7 +800,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface } if (! preg_match_all( - '#]*?)>.*?
([^<]+)
(.*?)#su', + '#
  • \s*]*?)>.*?
    ([^<]+)
    (.*?)#su', $html, $matches, PREG_SET_ORDER, @@ -680,7 +824,12 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface continue; } - $profileUrl = $this->resolveUrl(html_entity_decode($hrefMatch[1], ENT_QUOTES | ENT_HTML5, 'UTF-8'), $sourceUrl); + $href = html_entity_decode($hrefMatch[1], ENT_QUOTES | ENT_HTML5, 'UTF-8'); + if (! $this->looksLikeTeacherProfileUrl($href, null)) { + continue; + } + + $profileUrl = $this->resolveUrl($href, $sourceUrl); $dedupeKey = $profileUrl ?: ('name:'.md5($name)); if (isset($seen[$dedupeKey])) { continue; @@ -695,6 +844,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface if (preg_match('#研究方向：\s*([^<]+)#u', $tail, $fieldMatch)) { $researchField = CrawlAuthorParser::cleanText($fieldMatch[1]); } + $researchDirectionNames = $this->parseResearchDirectionNames($researchField ?? ''); $plain = trim($name.' '.($researchField ?? '').' '.($academicTitle ?? '').' '.($defaultCollege ?? '')); if (! $this->matchesKeywords($plain, $keywords)) { @@ -719,7 +869,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface keywords: $keywords, academicTitle: $academicTitle, platform: 'faculty_html_ra', - bio: $researchField, + bio: null, + phone: null, + researchDirectionNames: $researchDirectionNames !== [] ? $researchDirectionNames : null, ); } @@ -948,6 +1100,8 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface $academicTitle = CrawlAuthorParser::cleanText((string) ($art['exField2'] ?? '')); $researchField = CrawlAuthorParser::cleanText((string) ($art['exField1'] ?? '')); + $phone = $this->normalizePhone((string) ($art['phone'] ?? '')); + $researchDirectionNames = $this->parseResearchDirectionNames($researchField ?? ''); $plain = trim($name.' '.($researchField ?? '').' '.($academicTitle ?? '').' '.($defaultCollege ?? '')); if (! $this->matchesKeywords($plain, $keywords)) { continue; @@ -971,7 +1125,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface keywords: $keywords, academicTitle: $academicTitle, platform: 'faculty_html_nju_wp', - bio: $researchField, + bio: null, + phone: $phone !== '' ? $phone : null, + researchDirectionNames: $researchDirectionNames !== [] ? $researchDirectionNames : null, ); if (count($items) >= $maxResults) { @@ -1517,15 +1673,23 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface return true; } + if (preg_match('#/people/detail_new/\d+/?$#', $path)) { + return true; + } + + if (preg_match('#/cn/facultydetails/[^/]+/[^/]+/?$#', $path)) { + return true; + } + if (preg_match('#/c\d+a\d+/page\.htm$#', $path)) { return true; } - if (preg_match('#/(?:szll|zjzjs)/[^/]+\.(?:htm|html)$#', $path)) { + if (preg_match('#/(?:szll|zjzjs)(?:/[^/]+)*/[^/]+\.(?:htm|html)$#', $path)) { return true; } - if (preg_match('#^(?:szll|zjzjs)/[^/]+\.(?:htm|html)$#', $path)) { + if (preg_match('#^(?:szll|zjzjs)(?:/[^/]+)*/[^/]+\.(?:htm|html)$#', $path)) { return true; } @@ -1794,8 +1958,8 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface $lead = is_array($item->extra['lead_author'] ?? null) ? $item->extra['lead_author'] : []; $changed = false; - if (empty($lead['academic_title']) && preg_match('/\s*([^<]+?)\s*<\/em>/u', $html, $titleMatch)) { - $title = CrawlAuthorParser::cleanText($titleMatch[1]); + if (empty($lead['academic_title'])) { + $title = $this->extractAcademicTitleFromProfileHtml($html); if ($title !== null && $title !== '') { $lead['academic_title'] = $title; $changed = true; @@ -1803,7 +1967,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface } if (empty($lead['college']) && empty($lead['affiliation'])) { - $dept = $this->parseLabeledField($html, '所属二级机构'); + $dept = $this->parseLabeledField($html, '所属二级机构') + ?? $this->parseLabeledField($html, '所在单位') + ?? $this->parseLabeledField($html, '所在研究所'); if ($dept !== null && $dept !== '') { $lead['affiliation'] = $dept; $lead['college'] = $dept; @@ -1811,6 +1977,32 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface } } + if (empty($lead['phone'] ?? null) && empty($item->extra['phone'] ?? null)) { + $phone = $this->extractPhoneFromProfileHtml($html); + if ($phone !== null && $phone !== '') { + $lead['phone'] = $phone; + $changed = true; + } + } + + if (empty($lead['bio'] ?? null) && empty($item->extra['bio'] ?? null)) { + $bio = $this->extractBioFromProfileHtml($html); + if ($bio !== null && $bio !== '') { + $lead['bio'] = $bio; + $changed = true; + } + } + + $existingDirections = $item->extra['research_direction_names'] ?? ($lead['research_direction_names'] ?? null); + if (! is_array($existingDirections) || $existingDirections === []) { + $directionText = $this->extractResearchDirectionTextFromProfileHtml($html); + $directionNames = $this->parseResearchDirectionNames($directionText ?? ''); + if ($directionNames !== []) { + $lead['research_direction_names'] = $directionNames; + $changed = true; + } + } + if (! $changed) { return $item; } @@ -1823,6 +2015,15 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface if (! empty($lead['college'])) { $extra['college_name'] = $lead['college']; } + if (! empty($lead['phone'])) { + $extra['phone'] = $lead['phone']; + } + if (! empty($lead['bio'])) { + $extra['bio'] = $lead['bio']; + } + if (! empty($lead['research_direction_names']) && is_array($lead['research_direction_names'])) { + $extra['research_direction_names'] = $lead['research_direction_names']; + } $authorsParsed = $item->authorsParsed; if ($authorsParsed !== []) { @@ -1849,6 +2050,239 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface ); } + protected function extractAcademicTitleFromProfileHtml(string $html): ?string + { + if (preg_match('/\s*([^<]+?)\s*<\/em>/u', $html, $titleMatch)) { + $title = CrawlAuthorParser::cleanText($titleMatch[1]); + if ($title !== null && $title !== '') { + return $title; + } + } + + foreach (['职称', '职务'] as $label) { + $title = $this->parseLabeledField($html, $label); + if ($title !== null && $title !== '') { + return $title; + } + } + + if (preg_match('#
    ]*>.*?([^<]+)#su', $html, $match)) { + return CrawlAuthorParser::cleanText($match[1]); + } + + return null; + } + + protected function extractPhoneFromProfileHtml(string $html): ?string + { + $scoped = $this->profileContentHtml($html); + + $patterns = [ + '/办公电话[：:]\s*([0-9＋+\-—–()（）\s]{6,40})/u', + '/联系电话[：:]\s*([0-9＋+\-—–()（）\s]{6,40})/u', + '/details-tag">\s*联系电话\s*<\/span>\s*\s*([^<]+)/u', + '/电话[：:]\s*([0-9＋+\-—–()（）\s]{6,40})/u', + '/Tel(?:ephone)?[：:]\s*([0-9＋+\-—–()（）\s]{6,40})/iu', + '/Phone[：:]\s*([0-9＋+\-—–()（）\s]{6,40})/iu', + ]; + + foreach ($patterns as $pattern) { + if (preg_match($pattern, $scoped, $match)) { + $phone = $this->normalizePhone($match[1]); + if ($phone !== '' && ! $this->isNoisePhone($phone)) { + return $phone; + } + } + } + + return null; + } + + protected function extractBioFromProfileHtml(string $html): ?string + { + $patterns = [ + '#个人简介\s*
    \s*
    ]*>(.*?)
    #su', + '#class="h3">\s*个人简介\s*\s*
    (.*?)
    #su', + '#class="name">

    个人简介

    \s*
    (.*?)
    #su', + '#info-subChannel"[^>]*>\s*]*>\s*\s*个人简介\s*.*?
    ]*>(.*?)
    \s*#su', + '#个人简介[：:]\s*\s*
    ]*>(.*?)
    #su', + '#class="jj">个人简介[：:]\s*
    ]*>(.*?)
    #su', + '#class="sz-jj[^"]*"[^>]*>.*?
    ]*>(.*?)
    #su', + '#主要学术成绩]+>.*?]*>(.*?)#su', + ]; + + foreach ($patterns as $pattern) { + if (preg_match($pattern, $html, $match)) { + $bio = $this->htmlToPlain($match[1]); + $bio = CrawlAuthorParser::cleanText($bio); + if ($bio !== null && mb_strlen($bio) >= 20) { + return Str::limit($bio, 2000, ''); + } + } + } + + // 材料学院等：邮箱后的无标签长简介块 + if (preg_match_all('#
    ]*>(.*?)
    #su', $html, $blocks)) { + foreach ($blocks[1] as $block) { + $plain = $this->htmlToPlain($block); + if (preg_match('/电子邮箱|通讯地址|所属二级机构|办公电话|联系电话/u', $plain)) { + continue; + } + $plain = CrawlAuthorParser::cleanText($plain); + if ($plain !== null && mb_strlen($plain) >= 40) { + return Str::limit($plain, 2000, ''); + } + } + } + + return null; + } + + protected function extractResearchDirectionTextFromProfileHtml(string $html): ?string + { + $patterns = [ + // 材料学院 panel：研究方向 + ul>li + '#研究方向\s*.*?panel-body[^>]*>\s*
      (.*?)
    #su', + // 交大电气等：js-box-item name + txt + '#
    ]*>\s*
    \s*研究兴趣\s*
    \s*
    (.*?)
    #su', + '#
    ]*>\s*研究兴趣\s*(.*?)
    #su', + // 自动化/集成电路：tit 研究方向 + detail/txt + '#
    ]*>\s*(?:

    )?\s*研究方向\s*(?:

    )?\s*
    \s*
    ]*>\s*
    (.*?)
    #su', + '#研究方向
    \s*
    ]*>\s*
    (.*?)
    #su', + // 南大前沿：研究领域 + '#info-subChannel"[^>]*>\s*]*>\s*\s*研究领域\s*.*?
    ]*>(.*?)
    \s*
    #su', + // 南大集成电路：研究方向 + div.yj + '#研究方向[：:]\s*\s*
    (.*?)
    #su', + // 通用标签 + '#研究(?:方向|领域|兴趣)[：:]\s*]*>\s*([^<]{2,500})#u', + '#研究(?:方向|领域|兴趣)[：:]\s*([^<\n]{2,500})#u', + ]; + + foreach ($patterns as $pattern) { + if (! preg_match($pattern, $html, $match)) { + continue; + } + + $chunk = (string) $match[1]; + if (str_contains($chunk, ']*>(.*?)
    • #su', $chunk, $lis)) { + $parts = []; + foreach ($lis[1] as $li) { + $text = CrawlAuthorParser::cleanText($this->htmlToPlain($li)); + if ($text !== null && $text !== '') { + $parts[] = $text; + } + } + if ($parts !== []) { + return implode('、', $parts); + } + } + } + + if (preg_match_all('#]*>(.*?)

    #su', $chunk, $paragraphs) && count($paragraphs[1]) > 1) { + $parts = []; + $numberedParts = []; + foreach ($paragraphs[1] as $paragraph) { + $text = CrawlAuthorParser::cleanText($this->htmlToPlain($paragraph)); + if ($text === null || $text === '') { + continue; + } + $parts[] = $text; + if (preg_match('/^\d+[\.、．]/u', $text)) { + $numberedParts[] = preg_replace('/^\d+[\.、．]\s*/u', '', $text) ?? $text; + } + } + if ($numberedParts !== []) { + return implode('、', $numberedParts); + } + if ($parts !== []) { + return implode("\n", $parts); + } + } + + $text = CrawlAuthorParser::cleanText($this->htmlToPlain($chunk)); + if ($text !== null && $text !== '' && ! preg_match('/^(教育背景|工作经历|学术发表|项目资助)$/u', $text)) { + return $text; + } + } + + return null; + } + + /** + * @return list + */ + protected function parseResearchDirectionNames(string $direction): array + { + $direction = trim($direction); + if ($direction === '') { + return []; + } + + if (preg_match_all('/\d+[\.、．]\s*([^0-9\n]{2,80}?)(?=\s*\d+[\.、．]|[；;]|$)/u', $direction, $numbered)) { + $parts = array_values(array_unique(array_filter(array_map( + fn (string $part) => trim($part, " \t\n\r\0\x0B;；,，。"), + $numbered[1], + ), fn (string $part) => $part !== '' && mb_strlen($part) <= 80))); + if (count($parts) >= 2) { + return $parts; + } + } + + $parts = preg_split('/[、，,;；\/\|\r\n]+/u', $direction) ?: []; + + return array_values(array_unique(array_filter(array_map( + function (string $part): string { + $part = trim($part); + $part = preg_replace('/^\d+[\.、．]\s*/u', '', $part) ?? $part; + + return trim($part, " \t\n\r\0\x0B;；,，。."); + }, + $parts, + ), fn (string $part) => $part !== '' && mb_strlen($part) <= 80))); + } + + protected function normalizePhone(string $phone): string + { + $phone = html_entity_decode($phone, ENT_QUOTES | ENT_HTML5, 'UTF-8'); + $phone = trim(preg_replace('/\s+/u', ' ', $phone) ?? ''); + $phone = trim($phone, " \t\n\r\0\x0B;；,，"); + + return $phone; + } + + protected function isNoisePhone(string $phone): bool + { + $digits = preg_replace('/\D+/', '', $phone) ?? ''; + if (strlen($digits) < 6) { + return true; + } + + // 仅过滤明显占位号，保留 0 开头的座机号 + return (bool) preg_match('/^(0{6,}|1{6,}|123456+)/', $digits); + } + + protected function profileContentHtml(string $html): string + { + $scoped = preg_replace( + '#<(?:footer|div)[^>]*(?:class|id)=[\'"][^\'"]*(?:footer|foot-top|foot_nav|foot-logo|copyright)[^\'"]*[\'"][^>]*>.*$#isu', + '', + $html, + ); + if (! is_string($scoped) || $scoped === '') { + $scoped = $html; + } + + // 去掉页脚学院公共邮箱/电话残留 + $scoped = preg_replace( + '#<(?:p|div)[^>]*>[^<]*(?:Copyright|版权所有|备案号)[\s\S]*$#iu', + '', + $scoped, + ) ?? $scoped; + + return $scoped; + } + /** * @param list $keywords * @return list @@ -1937,6 +2371,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface /** * @param list $keywords + * @param list|null $researchDirectionNames */ protected function makeFacultyItem( string $externalKey, @@ -1950,17 +2385,22 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface ?string $academicTitle, string $platform, ?string $bio = null, + ?string $phone = null, + ?array $researchDirectionNames = null, ): CrawlItemDto { $college = $affiliation; + $directionNames = array_values(array_filter($researchDirectionNames ?? [])); $lead = [ 'name' => $name, 'email' => $email, + 'phone' => $phone, 'affiliation' => $college, 'college' => $college, 'university_name' => $universityName, 'academic_title' => $academicTitle, 'bio' => $bio, 'profile_url' => $profileUrl, + 'research_direction_names' => $directionNames !== [] ? $directionNames : null, ]; return new CrawlItemDto( @@ -1975,7 +2415,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface 'academic_title' => $academicTitle, 'college_name' => $college, 'bio' => $bio, + 'phone' => $phone, 'profile_url' => $profileUrl, + 'research_direction_names' => $directionNames !== [] ? $directionNames : null, 'lead_author' => $lead, 'keyword' => implode(' ', $keywords), ], diff --git a/app/Services/Crawl/CrawlImportService.php b/app/Services/Crawl/CrawlImportService.php index 5ca6a69..194b7a3 100644 --- a/app/Services/Crawl/CrawlImportService.php +++ b/app/Services/Crawl/CrawlImportService.php @@ -29,6 +29,7 @@ class CrawlImportService * papers_imported:int, * teacher_leads_imported:int, * teachers_imported:int, + * teachers_updated:int, * news_imported:int * } */ @@ -48,13 +49,14 @@ class CrawlImportService $query->whereIn('id', $itemIds); } - $items = $query->get(); + $items = $query->orderBy('id')->get(); $imported = 0; $skipped = 0; $failed = 0; $papersImported = 0; $teacherLeadsImported = 0; $teachersImported = 0; + $teachersUpdated = 0; $newsImported = 0; DB::transaction(function () use ( @@ -68,22 +70,26 @@ class CrawlImportService &$papersImported, &$teacherLeadsImported, &$teachersImported, + &$teachersUpdated, &$newsImported, ) { foreach ($items as $item) { try { - $id = match ($item->target_type) { - 'paper' => $this->importPaper($job, $item), + $result = match ($item->target_type) { + 'paper' => ['id' => $this->importPaper($job, $item), 'updated' => false], 'teacher_lead', 'teacher' => $this->importTeacher($job, $item, $teacherDefaults), - default => $this->importNews($job, $item, $newsDefaults), + default => ['id' => $this->importNews($job, $item, $newsDefaults), 'updated' => false], }; + $id = is_array($result) ? ($result['id'] ?? null) : $result; + $updated = is_array($result) ? (bool) ($result['updated'] ?? false) : false; + if ($id) { $item->update(['status' => 'imported', 'target_id' => $id]); $imported++; match ($item->target_type) { 'paper' => $papersImported++, 'teacher_lead' => $teacherLeadsImported++, - 'teacher' => $teachersImported++, + 'teacher' => $updated ? $teachersUpdated++ : $teachersImported++, default => $newsImported++, }; } else { @@ -116,6 +122,7 @@ class CrawlImportService 'papers_imported' => $papersImported, 'teacher_leads_imported' => $teacherLeadsImported, 'teachers_imported' => $teachersImported, + 'teachers_updated' => $teachersUpdated, 'news_imported' => $newsImported, ]; } @@ -172,8 +179,9 @@ class CrawlImportService /** * @param array{university_id?:int,department?:string,city?:string,research_direction_ids?:int[]} $defaults + * @return array{id:?int,updated:bool} */ - protected function importTeacher(CrawlJob $job, CrawlJobItem $item, array $defaults = []): ?int + protected function importTeacher(CrawlJob $job, CrawlJobItem $item, array $defaults = []): array { $payload = $item->payload ?? []; $lead = $payload['lead_author'] ?? null; @@ -188,18 +196,11 @@ class CrawlImportService $name = trim((string) ($lead['name'] ?? '')); if ($name === '') { - return null; + return ['id' => null, 'updated' => false]; } $email = CrawlAuthorParser::normalizeEmail($lead['email'] ?? null); - if ($email && Teacher::query()->where('email', $email)->exists()) { - $item->update(['status' => 'duplicate']); - - return null; - } - $phone = trim((string) ($lead['phone'] ?? $payload['phone'] ?? '')); - $academicTitle = trim((string) ($lead['academic_title'] ?? $payload['academic_title'] ?? '')); $collegeName = trim((string) ($lead['college'] ?? $lead['affiliation'] ?? $payload['college_name'] ?? '')); $defaultDepartment = trim((string) ($defaults['department'] ?? '')); @@ -222,10 +223,37 @@ class CrawlImportService } } - if ($this->teacherAlreadyExists($name, $email, $profileUrl, $universityId, $leadUniversityName, $collegeName)) { + $directionIds = $this->resolveTeacherDirectionIds($payload, $lead, $defaults); + + $existing = $this->findExistingTeacher( + $name, + $email, + $profileUrl, + $universityId, + $leadUniversityName, + $collegeName, + ); + + if ($existing) { + $updated = $this->fillEmptyTeacherFields($existing, [ + 'email' => $email, + 'phone' => $phone !== '' ? $phone : null, + 'title' => $academicTitle !== '' ? $academicTitle : null, + 'bio' => $bio !== '' ? $bio : null, + 'university_id' => $universityId, + 'department' => $collegeName !== '' ? $collegeName : null, + 'city' => $city, + 'profile_url' => $profileUrl !== '' ? $profileUrl : null, + 'research_direction_ids' => $directionIds, + ]); + + if ($updated) { + return ['id' => $existing->id, 'updated' => true]; + } + $item->update(['status' => 'duplicate']); - return null; + return ['id' => null, 'updated' => false]; } $sourceId = $this->resolveTeacherSourceId($item->target_type); @@ -233,8 +261,6 @@ class CrawlImportService throw new \RuntimeException('老师库字典未配置'); } - $statusId = null; - $remarkParts = [ match ($item->target_type) { 'teacher_lead' => '论文库入库', @@ -272,23 +298,11 @@ class CrawlImportService 'email' => $email, 'phone' => $phone !== '' ? $phone : null, 'source_dict_item_id' => $sourceId, - 'status_dict_item_id' => $statusId, + 'status_dict_item_id' => null, 'library_status' => $libraryStatus, 'remark' => implode('；', $remarkParts), ]); - $directionIds = []; - if (! empty($defaults['research_direction_ids']) && is_array($defaults['research_direction_ids'])) { - $directionIds = array_map('intval', $defaults['research_direction_ids']); - } else { - $directionNames = $payload['research_direction_names'] ?? $lead['research_direction_names'] ?? []; - if (is_string($directionNames)) { - $directionNames = preg_split('/[、，,;；\/]+/u', $directionNames) ?: []; - } - if (is_array($directionNames) && $directionNames !== []) { - $directionIds = app(ResearchDirectionResolver::class)->resolveIds([], $directionNames); - } - } if ($directionIds !== []) { $teacher->researchDirections()->sync($directionIds); } @@ -304,7 +318,162 @@ class CrawlImportService } } - return $teacher->id; + return ['id' => $teacher->id, 'updated' => false]; + } + + /** + * @param array $payload + * @param array $lead + * @param array{university_id?:int,department?:string,city?:string,research_direction_ids?:int[]} $defaults + * @return list + */ + protected function resolveTeacherDirectionIds(array $payload, array $lead, array $defaults): array + { + if (! empty($defaults['research_direction_ids']) && is_array($defaults['research_direction_ids'])) { + return array_values(array_unique(array_map('intval', $defaults['research_direction_ids']))); + } + + $directionNames = $payload['research_direction_names'] ?? $lead['research_direction_names'] ?? []; + if (is_string($directionNames)) { + $directionNames = preg_split('/[、，,;；\/]+/u', $directionNames) ?: []; + } + if (! is_array($directionNames) || $directionNames === []) { + return []; + } + + return app(ResearchDirectionResolver::class)->resolveIds([], $directionNames); + } + + /** + * 仅补空字段：已有值不覆盖。 + * + * @param array{ + * email?:?string, + * phone?:?string, + * title?:?string, + * bio?:?string, + * university_id?:?int, + * department?:?string, + * city?:?string, + * profile_url?:?string, + * research_direction_ids?:list + * } $incoming + */ + public function fillEmptyTeacherFields(Teacher $teacher, array $incoming): bool + { + $updates = []; + $changed = false; + + $email = CrawlAuthorParser::normalizeEmail($incoming['email'] ?? null); + if ($this->isBlankTeacherValue($teacher->email) && $email) { + $emailTaken = Teacher::query() + ->where('email', $email) + ->where('id', '!=', $teacher->id) + ->exists(); + if (! $emailTaken) { + $updates['email'] = $email; + } + } + + $phone = trim((string) ($incoming['phone'] ?? '')); + if ($this->isBlankTeacherValue($teacher->phone) && $phone !== '') { + $updates['phone'] = $phone; + } + + $title = trim((string) ($incoming['title'] ?? '')); + if ($this->isBlankTeacherValue($teacher->title) && $title !== '') { + $updates['title'] = $title; + } + + $bio = trim((string) ($incoming['bio'] ?? '')); + if ($this->isBlankTeacherValue($teacher->bio) && $bio !== '') { + $updates['bio'] = $bio; + } + + $department = trim((string) ($incoming['department'] ?? '')); + if ($this->isBlankTeacherValue($teacher->department) && $department !== '') { + $updates['department'] = $department; + } + + $city = trim((string) ($incoming['city'] ?? '')); + if ($this->isBlankTeacherValue($teacher->city) && $city !== '') { + $updates['city'] = $city; + } + + $universityId = isset($incoming['university_id']) ? (int) $incoming['university_id'] : 0; + if (! $teacher->university_id && $universityId > 0) { + $updates['university_id'] = $universityId; + } + + $profileUrl = trim((string) ($incoming['profile_url'] ?? '')); + if ($profileUrl !== '') { + $remark = (string) ($teacher->remark ?? ''); + if ($remark === '' || ! str_contains($remark, $profileUrl)) { + $updates['remark'] = trim($remark.($remark !== '' ? '；' : '').'主页：'.$profileUrl, '；'); + } + } + + if ($updates !== []) { + $teacher->fill($updates); + $teacher->save(); + $changed = true; + } + + $directionIds = array_values(array_unique(array_filter(array_map( + 'intval', + $incoming['research_direction_ids'] ?? [], + )))); + if ($directionIds !== [] && $teacher->researchDirections()->count() === 0) { + $teacher->researchDirections()->sync($directionIds); + $changed = true; + } + + return $changed; + } + + protected function isBlankTeacherValue(mixed $value): bool + { + $text = trim((string) ($value ?? '')); + + return $text === '' || $text === '待补充'; + } + + protected function findExistingTeacher( + string $name, + ?string $email, + ?string $profileUrl, + ?int $universityId, + string $leadUniversityName, + ?string $collegeName, + ): ?Teacher { + if ($email) { + $byEmail = Teacher::query()->where('email', $email)->first(); + if ($byEmail) { + return $byEmail; + } + } + + if ($profileUrl !== null && $profileUrl !== '') { + $byProfile = Teacher::query()->where('remark', 'like', '%'.$profileUrl.'%')->first(); + if ($byProfile) { + return $byProfile; + } + } + + $dupQuery = Teacher::query()->where('name', $name); + if ($collegeName !== null && $collegeName !== '') { + $dupQuery->where('department', $collegeName); + } + if ($universityId) { + $dupQuery->where('university_id', $universityId); + } elseif ($leadUniversityName !== '') { + $dupQuery->where(function ($q) use ($leadUniversityName) { + $q->where('university_text', $leadUniversityName) + ->orWhere('university_text', 'like', $leadUniversityName.'%'); + }); + } + + return $dupQuery->first(); } /** @@ -430,42 +599,6 @@ class CrawlImportService } } - protected function teacherAlreadyExists( - string $name, - ?string $email, - ?string $profileUrl, - ?int $universityId, - string $leadUniversityName, - ?string $collegeName, - ): bool { - if ($email && Teacher::query()->where('email', $email)->exists()) { - return true; - } - - if ($email) { - return false; - } - - if ($profileUrl !== '') { - return Teacher::query()->where('remark', 'like', '%'.$profileUrl.'%')->exists(); - } - - $dupQuery = Teacher::query()->where('name', $name); - if ($collegeName !== '') { - $dupQuery->where('department', $collegeName); - } - if ($universityId) { - $dupQuery->where('university_id', $universityId); - } elseif ($leadUniversityName !== '') { - $dupQuery->where(function ($q) use ($leadUniversityName) { - $q->where('university_text', $leadUniversityName) - ->orWhere('university_text', 'like', $leadUniversityName.'%'); - }); - } - - return $dupQuery->exists(); - } - protected function resolveUniversityId(string $name): ?int { $name = trim($name); diff --git a/app/Services/Crawl/CrawlJobRunnerService.php b/app/Services/Crawl/CrawlJobRunnerService.php index b8b0ec4..036c40e 100644 --- a/app/Services/Crawl/CrawlJobRunnerService.php +++ b/app/Services/Crawl/CrawlJobRunnerService.php @@ -7,7 +7,6 @@ use App\Models\CrawlJobItem; use App\Models\CrawlSource; use App\Models\News; use App\Models\Paper; -use App\Models\Teacher; class CrawlJobRunnerService { @@ -163,36 +162,8 @@ class CrawlJobRunnerService protected function persistTeacherItem(CrawlJob $job, CrawlSource $source, CrawlItemDto $dto): bool { $lead = $dto->extra['lead_author'] ?? CrawlAuthorParser::leadAuthor($dto->authors, $dto->authorsParsed); - $email = is_array($lead) ? ($lead['email'] ?? null) : null; - $status = 'preview'; - if ($email && Teacher::query()->where('email', $email)->exists()) { - $status = 'duplicate'; - } elseif (is_array($lead)) { - $leadName = trim((string) ($lead['name'] ?? '')); - $leadUniversity = trim((string) ($lead['university_name'] ?? '')); - $profileUrl = trim((string) ($lead['profile_url'] ?? $dto->canonicalUrl ?? '')); - $collegeName = trim((string) ($lead['college'] ?? $lead['affiliation'] ?? $dto->extra['college_name'] ?? '')); - if ($leadName !== '' && ! $email) { - if ($profileUrl !== '' && Teacher::query()->where('remark', 'like', '%'.$profileUrl.'%')->exists()) { - $status = 'duplicate'; - } else { - $dup = Teacher::query()->where('name', $leadName); - if ($collegeName !== '') { - $dup->where('department', $collegeName); - } - if ($leadUniversity !== '') { - $dup->where(function ($q) use ($leadUniversity) { - $q->where('university_text', $leadUniversity) - ->orWhere('university_text', 'like', $leadUniversity.'%'); - }); - } - if ($dup->exists()) { - $status = 'duplicate'; - } - } - } - } + // 已存在老师也先进入 preview，入库时仅补空字段；无空字段可补再标 duplicate CrawlJobItem::query()->updateOrCreate( [ 'crawl_job_id' => $job->id, @@ -209,10 +180,11 @@ class CrawlJobRunnerService 'college_name' => $dto->extra['college_name'] ?? (is_array($lead) ? ($lead['college'] ?? $lead['affiliation'] ?? null) : null), 'profile_url' => $dto->extra['profile_url'] ?? $dto->canonicalUrl, 'phone' => $dto->extra['phone'] ?? (is_array($lead) ? ($lead['phone'] ?? null) : null), + 'bio' => $dto->extra['bio'] ?? (is_array($lead) ? ($lead['bio'] ?? null) : null), 'research_direction_names' => $dto->extra['research_direction_names'] ?? (is_array($lead) ? ($lead['research_direction_names'] ?? null) : null), ], - 'status' => $status, + 'status' => 'preview', 'target_type' => 'teacher', 'source_name' => $source->name, 'target_id' => null, diff --git a/config/crawl.php b/config/crawl.php index 712d81a..2458ff9 100644 --- a/config/crawl.php +++ b/config/crawl.php @@ -29,10 +29,10 @@ return [ ], 'faculty' => [ - /** 列表项无邮箱时，是否请求教师主页补全邮箱 */ + /** 是否请求教师主页补全邮箱/电话/简介/研究方向 */ 'profile_email_enrich_enabled' => (bool) env('FACULTY_PROFILE_EMAIL_ENRICH', true), - /** 单次任务最多补全主页数（其余仍入库，仅无邮箱） */ - 'profile_enrich_max' => (int) env('FACULTY_PROFILE_ENRICH_MAX', 32), + /** 单次任务最多补全主页数（缺邮箱/电话/简介/研究方向时访问详情页） */ + 'profile_enrich_max' => (int) env('FACULTY_PROFILE_ENRICH_MAX', 200), 'profile_http_timeout_seconds' => (int) env('FACULTY_PROFILE_HTTP_TIMEOUT', 10), /** 并发请求教师主页数 */ 'profile_enrich_pool_size' => (int) env('FACULTY_PROFILE_ENRICH_POOL', 8), diff --git a/tests/Unit/CrawlImportTeacherFillEmptyTest.php b/tests/Unit/CrawlImportTeacherFillEmptyTest.php new file mode 100644 index 0000000..a10923a --- /dev/null +++ b/tests/Unit/CrawlImportTeacherFillEmptyTest.php @@ -0,0 +1,76 @@ +create([ + 'name' => '补空字段测试老师', + 'university_id' => null, + 'department' => '计算机学院', + 'bio' => null, + 'city' => '待补充', + 'title' => '待补充', + 'email' => null, + 'phone' => null, + 'library_status' => TeacherLibraryStatus::Active, + 'remark' => '高校抓取入库', + ]); + + $direction = ResearchDirection::query()->firstOrCreate( + ['name' => '补空字段测试方向'], + ['sort' => 0, 'status' => 1, 'remark' => 'test'], + ); + + try { + $service = app(CrawlImportService::class); + $changed = $service->fillEmptyTeacherFields($teacher, [ + 'email' => 'fill-empty-test@sjtu.edu.cn', + 'phone' => '021-12345678', + 'title' => '教授', + 'bio' => '长期从事操作系统研究。', + 'city' => '上海', + 'profile_url' => 'https://www.cs.sjtu.edu.cn/jiaoshiml/fill-empty-test.html', + 'research_direction_ids' => [$direction->id], + ]); + + $this->assertTrue($changed); + $teacher->refresh(); + $this->assertSame('fill-empty-test@sjtu.edu.cn', $teacher->email); + $this->assertSame('021-12345678', $teacher->phone); + $this->assertSame('教授', $teacher->title); + $this->assertSame('长期从事操作系统研究。', $teacher->bio); + $this->assertSame('上海', $teacher->city); + $this->assertStringContainsString('fill-empty-test.html', (string) $teacher->remark); + $this->assertSame([$direction->id], $teacher->researchDirections()->pluck('research_directions.id')->all()); + + // 已有值不应被覆盖 + $changedAgain = $service->fillEmptyTeacherFields($teacher, [ + 'email' => 'other@sjtu.edu.cn', + 'phone' => '021-87654321', + 'title' => '副教授', + 'bio' => '另一段简介', + 'city' => '北京', + 'research_direction_ids' => [$direction->id], + ]); + $this->assertFalse($changedAgain); + $teacher->refresh(); + $this->assertSame('fill-empty-test@sjtu.edu.cn', $teacher->email); + $this->assertSame('021-12345678', $teacher->phone); + $this->assertSame('教授', $teacher->title); + $this->assertSame('长期从事操作系统研究。', $teacher->bio); + $this->assertSame('上海', $teacher->city); + } finally { + $teacher->researchDirections()->detach(); + $teacher->forceDelete(); + } + } +} diff --git a/tests/Unit/FacultyListHtmlAdapterTest.php b/tests/Unit/FacultyListHtmlAdapterTest.php index b126e2a..735d273 100644 --- a/tests/Unit/FacultyListHtmlAdapterTest.php +++ b/tests/Unit/FacultyListHtmlAdapterTest.php @@ -183,6 +183,17 @@ HTML; $html = <<<'HTML'

    陈军

    教授
    所属二级机构：塑性成形技术与装备研究院
    +
    电子邮箱：jun_chen@sjtu.edu.cn
    +
    +
    研究方向
    +
    +
    +
    +
      +
    • 轻质高强薄板塑性成形与数控渐进成形
      • +
    • 塑性变形力学模型及其多物理场数值仿真
      • +
    +
    HTML; $adapter = new FacultyListHtmlAdapter; @@ -208,6 +219,170 @@ HTML; $this->assertSame('教授', $item->extra['lead_author']['academic_title']); $this->assertSame('塑性成形技术与装备研究院', $item->extra['college_name']); + $this->assertSame( + ['轻质高强薄板塑性成形与数控渐进成形', '塑性变形力学模型及其多物理场数值仿真'], + $item->extra['research_direction_names'], + ); + } + + public function test_apply_profile_metadata_from_see_detail_page(): void + { + $html = <<<'HTML' +
    +

    办公电话：021-34204298

    +

    电子邮件：aiqian@sjtu.edu.cn

    +
    +
    +
    +
    研究兴趣
    +

    电力系统元件建模, 电能质量, 分布式发电

    +
    +
    +
    教育背景
    +

    1994.09 –1998.07清华大学

    +
    +
    +

    邮箱：see@sjtu.edu.cn

    +HTML; + + $adapter = new FacultyListHtmlAdapter; + $method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem'); + $method->setAccessible(true); + $item = $method->invoke( + $adapter, + new \App\Services\Crawl\CrawlItemDto( + externalId: 'faculty:see', + title: '艾芊', + canonicalUrl: 'https://see.sjtu.edu.cn/jiaoshiml/aiqian.html', + extra: ['lead_author' => ['name' => '艾芊']], + ), + $html, + ); + + $this->assertSame('021-34204298', $item->extra['phone']); + $this->assertSame(['电力系统元件建模', '电能质量', '分布式发电'], $item->extra['research_direction_names']); + + $emailMethod = new \ReflectionMethod($adapter, 'extractEmailFromProfileHtml'); + $emailMethod->setAccessible(true); + $this->assertSame('aiqian@sjtu.edu.cn', $emailMethod->invoke($adapter, $html)); + } + + public function test_apply_profile_metadata_from_cs_and_frontier_pages(): void + { + $adapter = new FacultyListHtmlAdapter; + $method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem'); + $method->setAccessible(true); + + $csHtml = <<<'HTML' +

    邮箱：byzang@sjtu.edu.cn

    +
    +
    +

    个人简介

    +

    上海交通大学特聘教授，主要研究领域为操作系统、分布式系统与系统安全。

    +
    +
    +HTML; + $csItem = $method->invoke( + $adapter, + new \App\Services\Crawl\CrawlItemDto( + externalId: 'faculty:cs', + title: '臧斌宇', + canonicalUrl: 'https://www.cs.sjtu.edu.cn/jiaoshiml/zangbinyu.html', + extra: ['lead_author' => ['name' => '臧斌宇']], + ), + $csHtml, + ); + $this->assertStringContainsString('操作系统', (string) $csItem->extra['bio']); + + $frontierHtml = <<<'HTML' +
    + 电子邮件 + baoming@nju.edu.cn +
    +
    + 联系电话 + 0512-68768786 +
    +
    +
    研究领域
    +

    微纳材料与微纳系统

    +
    +
    +
    个人简介
    +

    2006.09-2010.07 厦门大学，本科；长期从事微纳材料研究。

    +
    +HTML; + $frontierItem = $method->invoke( + $adapter, + new \App\Services\Crawl\CrawlItemDto( + externalId: 'faculty:frontier', + title: '王保明', + canonicalUrl: 'https://frontier.nju.edu.cn/85/ef/c59286a689647/page.htm', + extra: ['lead_author' => ['name' => '王保明']], + ), + $frontierHtml, + ); + $this->assertSame('0512-68768786', $frontierItem->extra['phone']); + $this->assertSame(['微纳材料与微纳系统'], $frontierItem->extra['research_direction_names']); + $this->assertStringContainsString('厦门大学', (string) $frontierItem->extra['bio']); + } + + public function test_extracts_ra_teacher_list_with_research_directions(): void + { + $html = <<<'HTML' +专职教师-南京大学机器人与自动化学院 + +HTML; + + $adapter = new FacultyListHtmlAdapter; + $method = new \ReflectionMethod($adapter, 'extractFromHtml'); + $method->setAccessible(true); + $items = $method->invoke($adapter, $html, [], 'https://ra.nju.edu.cn/szll/zzjs/index.html'); + + $this->assertCount(1, $items); + $this->assertSame('周克敏', $items[0]->title); + $this->assertSame('http://ra.nju.edu.cn/szll/zzjs/20250901/i335910.html', $items[0]->canonicalUrl); + $this->assertSame(['鲁棒控制', '多目标优化控制'], $items[0]->extra['research_direction_names']); + } + + public function test_item_needs_profile_enrich_when_research_missing(): void + { + $adapter = new FacultyListHtmlAdapter; + $method = new \ReflectionMethod($adapter, 'itemNeedsProfileEnrich'); + $method->setAccessible(true); + + $item = new \App\Services\Crawl\CrawlItemDto( + externalId: 'faculty:x', + title: '张三', + canonicalUrl: 'https://see.sjtu.edu.cn/jiaoshiml/zhangsan.html', + extra: [ + 'lead_author' => [ + 'name' => '张三', + 'email' => 'zhangsan@sjtu.edu.cn', + ], + ], + ); + + $this->assertTrue($method->invoke($adapter, $item)); + } + + public function test_resolve_profile_enrich_max_caps_large_batches(): void + { + $adapter = new FacultyListHtmlAdapter; + $method = new \ReflectionMethod($adapter, 'resolveProfileEnrichMax'); + $method->setAccessible(true); + + $this->assertSame(200, $method->invoke($adapter, [], 500)); + $this->assertSame(10, $method->invoke($adapter, ['profile_enrich_max' => 10], 500)); + $this->assertSame(0, $method->invoke($adapter, ['skip_profile_enrich' => true], 500)); } public function test_extracts_sais_js_list_from_ajax_content(): void @@ -270,17 +445,6 @@ HTML; $this->assertSame('https://www.cs.sjtu.edu.cn/jiaoshiml/chenhaibo.html', $items[1]->canonicalUrl); } - public function test_resolve_profile_enrich_max_caps_large_batches(): void - { - $adapter = new FacultyListHtmlAdapter; - $method = new \ReflectionMethod($adapter, 'resolveProfileEnrichMax'); - $method->setAccessible(true); - - $this->assertSame(32, $method->invoke($adapter, [], 500)); - $this->assertSame(10, $method->invoke($adapter, ['profile_enrich_max' => 10], 500)); - $this->assertSame(0, $method->invoke($adapter, ['skip_profile_enrich' => true], 500)); - } - public function test_response_body_from_pool_result_ignores_connection_exception(): void { $adapter = new FacultyListHtmlAdapter;