wangyifang
/
slake-school-service
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

老师库研究方向

Browse Source
master
lion 2 days ago
parent 3e063bed15
commit 2c5d6762a9
7 changed files with 958 additions and 144 deletions
Show Stats Download Patch File Download Diff File

37
app/Http/Controllers/Admin/CrawlJobController.php
Unescape View File

@ -227,6 +227,8 @@ class CrawlJobController extends Controller
'imported_primary' => $result['imported_primary'],
'skipped' => $result['skipped'],
'failed' => $result['failed'],
'teachers_imported' => $result['teachers_imported'] ?? 0,
'teachers_updated' => $result['teachers_updated'] ?? 0,
'items_imported' => $job->fresh()->items_imported,
], $this->buildImportSuccessMessage($job, $result));
}
@ -432,7 +434,9 @@ class CrawlJobController extends Controller
}
if ($job->target_type === 'teacher') {
$imported = (int) ($importResult['teachers_imported'] ?? 0);
$created = (int) ($importResult['teachers_imported'] ?? 0);
$updated = (int) ($importResult['teachers_updated'] ?? 0);
$imported = $created + $updated;
$duplicateCount = $this->countTeacherDuplicateItems($job);
$summary = sprintf(
'已从 %s 抓取 %d 位老师，已入库 %d 位老师',
@ -440,8 +444,18 @@ class CrawlJobController extends Controller
$fetched,
$imported,
);
if ($created > 0 || $updated > 0) {
$parts = [];
if ($created > 0) {
$parts[] = sprintf('新建 %d', $created);
}
if ($updated > 0) {
$parts[] = sprintf('补全 %d', $updated);
}
$summary .= '（'.implode('，', $parts).'）';
}
if ($duplicateCount > 0) {
$summary .= sprintf('，跳过 %d 位（老师库中已存在）', $duplicateCount);
$summary .= sprintf('，跳过 %d 位（已存在且无需补全）', $duplicateCount);
}
$failedCount = (int) ($importResult['failed'] ?? 0);
if ($failedCount > 0) {
@ -508,13 +522,26 @@ class CrawlJobController extends Controller
}
if ($job->target_type === 'teacher') {
$teachers = (int) ($importResult['teachers_imported'] ?? 0);
$created = (int) ($importResult['teachers_imported'] ?? 0);
$updated = (int) ($importResult['teachers_updated'] ?? 0);
$teachers = $created + $updated;
$duplicateCount = $this->countTeacherDuplicateItems($job);
$message = "抓取完成，已入库 {$teachers} 位老师";
if ($created > 0 || $updated > 0) {
$parts = [];
if ($created > 0) {
$parts[] = "新建 {$created}";
}
if ($updated > 0) {
$parts[] = "补全 {$updated}";
}
$message .= '（'.implode('，', $parts).'）';
}
if ($duplicateCount > 0) {
return "抓取完成，已入库 {$teachers} 位老师，跳过 {$duplicateCount} 位";
$message .= "，跳过 {$duplicateCount} 位";
}
return "抓取完成，已入库 {$teachers} 位老师";
return $message;
}
$news = (int) ($importResult['news_imported'] ?? 0);

494
app/Services/Crawl/Adapters/FacultyListHtmlAdapter.php
Unescape View File

@ -183,7 +183,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
if ($enrichBudget <= 0) {
break;
}
if ($this->itemHasEmail($item) || ! $item->canonicalUrl) {
if (! $this->itemNeedsProfileEnrich($item)) {
continue;
}
$fetchMap[$index] = $item;
@ -215,9 +215,11 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
foreach ($batchPending as $index => $item) {
$body = $this->responseBodyFromPoolResult($responses[(string) $index] ?? null);
if ($body !== null) {
$email = $this->extractEmailFromProfileHtml($body);
if ($email) {
$item = $this->applyEmailToItem($item, $email);
if (! $this->itemHasEmail($item)) {
$email = $this->extractEmailFromProfileHtml($body);
if ($email) {
$item = $this->applyEmailToItem($item, $email);
}
}
$item = $this->applyProfileMetadataToItem($item, $body);
}
@ -229,7 +231,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
foreach ($items as $index => $item) {
if (isset($fetchedBodies[$index])) {
$result[] = $fetchedBodies[$index];
} elseif (! $this->itemHasEmail($item) && $item->canonicalUrl) {
} elseif ($this->itemNeedsProfileEnrich($item)) {
$result[] = $this->markItemProfileEnrichSkipped($item);
} else {
$result[] = $item;
@ -239,6 +241,62 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
return $result;
}
protected function itemNeedsProfileEnrich(CrawlItemDto $item): bool
{
if (! $item->canonicalUrl) {
return false;
}
return ! $this->itemHasEmail($item)
|| ! $this->itemHasPhone($item)
|| ! $this->itemHasResearchDirections($item)
|| ! $this->itemHasBio($item);
}
protected function itemHasPhone(CrawlItemDto $item): bool
{
$lead = $item->extra['lead_author'] ?? null;
$phone = is_array($lead) ? trim((string) ($lead['phone'] ?? '')) : '';
if ($phone === '') {
$phone = trim((string) ($item->extra['phone'] ?? ''));
}
return $phone !== '';
}
protected function itemHasResearchDirections(CrawlItemDto $item): bool
{
$names = $item->extra['research_direction_names'] ?? null;
if (is_array($names) && $names !== []) {
return true;
}
$lead = $item->extra['lead_author'] ?? null;
if (is_array($lead)) {
$leadNames = $lead['research_direction_names'] ?? null;
if (is_array($leadNames) && $leadNames !== []) {
return true;
}
}
return false;
}
protected function itemHasBio(CrawlItemDto $item): bool
{
$bio = trim((string) ($item->extra['bio'] ?? ''));
if ($bio !== '') {
return true;
}
$lead = $item->extra['lead_author'] ?? null;
if (is_array($lead) && trim((string) ($lead['bio'] ?? '')) !== '') {
return true;
}
return false;
}
/**
* @param array<string, mixed> $params
*/
@ -248,9 +306,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
return 0;
}
$configured = (int) ($params['profile_enrich_max'] ?? config('crawl.faculty.profile_enrich_max', 32));
$configured = (int) ($params['profile_enrich_max'] ?? config('crawl.faculty.profile_enrich_max', 200));
return max(0, min($itemCount, min(200, $configured)));
return max(0, min($itemCount, min(500, $configured)));
}
/**
@ -264,7 +322,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
protected function markItemProfileEnrichSkipped(CrawlItemDto $item): CrawlItemDto
{
if ($this->itemHasEmail($item)) {
if (! $this->itemNeedsProfileEnrich($item)) {
return $item;
}
@ -349,17 +407,22 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
protected function extractEmailFromProfileHtml(string $html): ?string
{
$scoped = $this->profileContentHtml($html);
$labeledPatterns = [
'/电子邮箱[：:]\s*<\/strong>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子邮箱[：:]\s*<\/(?:strong|span|b)>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子信箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/E-?mail[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/iu',
'/邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子邮件[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/E-?mail[：:]\s*(?:<[^>]+>\s*)*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/iu',
'/details-tag">\s*电子邮件\s*<\/span>\s*<span class="details-con">\s*(?:<a[^>]*>)?\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/电子邮箱[：:]\s*<\/span>\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/邮箱[：:]\s*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/u',
'/mailto:([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/i',
];
foreach ($labeledPatterns as $pattern) {
if (preg_match($pattern, $html, $match)) {
if (preg_match($pattern, $scoped, $match)) {
$email = CrawlAuthorParser::normalizeEmail($match[1]);
if ($email && ! $this->isNoiseEmail($email)) {
return $email;
@ -370,7 +433,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
$candidates = [];
if (preg_match_all(
'#([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})#',
$html,
$scoped,
$emailMatches,
)) {
foreach ($emailMatches[1] as $raw) {
@ -398,10 +461,16 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
protected function isNoiseEmail(string $email): bool
{
return (bool) preg_match(
'/^(noreply|no-reply|admin|webmaster|postmaster|root|support|service|info|contact)@/i',
$email,
);
$local = strtolower((string) strstr($email, '@', true));
if (preg_match(
'/^(noreply|no-reply|admin|webmaster|postmaster|root|support|service|info|contact|see|scs|soai|icisee|sais|smse|faculty|office|dean)$/i',
$local,
)) {
return true;
}
return false;
}
protected function fetchHtml(string $url): string
@ -501,6 +570,11 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
return $items;
}
$items = $this->extractFromSoaiFacultyList($html, $keywords, $sourceUrl);
if ($items !== []) {
return $items;
}
$items = $this->extractFromVsbFacultyTable($html, $keywords, $sourceUrl);
if ($items !== []) {
return $items;
@ -519,6 +593,76 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
return $this->extractFromStaffPanelList($html, $keywords, $sourceUrl);
}
/**
* 交大人工智能学院：/cn/facultydetails/... 卡片列表。
*
* @param list<string> $keywords
* @return list<CrawlItemDto>
*/
protected function extractFromSoaiFacultyList(string $html, array $keywords, string $sourceUrl): array
{
if (! preg_match_all(
'#<a\b([^>]*?/cn/facultydetails/[^\'"]+[^>]*)>([^<]+)</a>#u',
$html,
$matches,
PREG_SET_ORDER,
)) {
return [];
}
$pageUniversity = $this->inferUniversityFromSource($sourceUrl, $html);
$defaultCollege = $this->inferCollegeFromPageTitle($html) ?? '人工智能学院';
$items = [];
$seen = [];
foreach ($matches as $match) {
$attrs = (string) $match[1];
$name = CrawlAuthorParser::cleanText($match[2]) ?? '';
if ($name === '' || ! $this->looksLikePersonName($name)) {
continue;
}
if (! preg_match('#\bhref=[\'"]([^\'"]+)[\'"]#u', $attrs, $hrefMatch)) {
continue;
}
$href = html_entity_decode($hrefMatch[1], ENT_QUOTES | ENT_HTML5, 'UTF-8');
$profileUrl = $this->resolveUrl($href, $sourceUrl);
$dedupeKey = $profileUrl ?: ('name:'.md5($name));
if (isset($seen[$dedupeKey])) {
continue;
}
$plain = trim($name.' '.($defaultCollege ?? ''));
if (! $this->matchesKeywords($plain, $keywords)) {
continue;
}
$academicTitle = null;
$windowStart = max(0, (int) strpos($html, $match[0]) - 50);
$window = substr($html, $windowStart, 500);
if (preg_match('#职称[：:]\s*([^<]+)#u', $window, $titleMatch)) {
$academicTitle = CrawlAuthorParser::cleanText($titleMatch[1]);
}
$seen[$dedupeKey] = true;
$items[] = $this->makeFacultyItem(
externalKey: 'faculty:'.md5($dedupeKey),
name: $name,
profileUrl: $profileUrl,
email: null,
affiliation: $defaultCollege,
universityName: $pageUniversity ?? '上海交通大学',
summary: $defaultCollege ? '单位：'.$defaultCollege : null,
keywords: $keywords,
academicTitle: $academicTitle,
platform: 'faculty_html_soai',
bio: null,
);
}
return $items;
}
/**
* 南大 Sudy CMS：ul.news_list 内 news_title / news_title1 链接（frontier、ic 等）。
*
@ -656,7 +800,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
}
if (! preg_match_all(
'#<a\b([^>]*?)>.*?<div class="xm">([^<]+)</div>(.*?)</a>#su',
'#<li>\s*<a\b([^>]*?)>.*?<div class="xm">([^<]+)</div>(.*?)</a>#su',
$html,
$matches,
PREG_SET_ORDER,
@ -680,7 +824,12 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
continue;
}
$profileUrl = $this->resolveUrl(html_entity_decode($hrefMatch[1], ENT_QUOTES | ENT_HTML5, 'UTF-8'), $sourceUrl);
$href = html_entity_decode($hrefMatch[1], ENT_QUOTES | ENT_HTML5, 'UTF-8');
if (! $this->looksLikeTeacherProfileUrl($href, null)) {
continue;
}
$profileUrl = $this->resolveUrl($href, $sourceUrl);
$dedupeKey = $profileUrl ?: ('name:'.md5($name));
if (isset($seen[$dedupeKey])) {
continue;
@ -695,6 +844,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
if (preg_match('#研究方向：\s*<span>([^<]+)</span>#u', $tail, $fieldMatch)) {
$researchField = CrawlAuthorParser::cleanText($fieldMatch[1]);
}
$researchDirectionNames = $this->parseResearchDirectionNames($researchField ?? '');
$plain = trim($name.' '.($researchField ?? '').' '.($academicTitle ?? '').' '.($defaultCollege ?? ''));
if (! $this->matchesKeywords($plain, $keywords)) {
@ -719,7 +869,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
keywords: $keywords,
academicTitle: $academicTitle,
platform: 'faculty_html_ra',
bio: $researchField,
bio: null,
phone: null,
researchDirectionNames: $researchDirectionNames !== [] ? $researchDirectionNames : null,
);
}
@ -948,6 +1100,8 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
$academicTitle = CrawlAuthorParser::cleanText((string) ($art['exField2'] ?? ''));
$researchField = CrawlAuthorParser::cleanText((string) ($art['exField1'] ?? ''));
$phone = $this->normalizePhone((string) ($art['phone'] ?? ''));
$researchDirectionNames = $this->parseResearchDirectionNames($researchField ?? '');
$plain = trim($name.' '.($researchField ?? '').' '.($academicTitle ?? '').' '.($defaultCollege ?? ''));
if (! $this->matchesKeywords($plain, $keywords)) {
continue;
@ -971,7 +1125,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
keywords: $keywords,
academicTitle: $academicTitle,
platform: 'faculty_html_nju_wp',
bio: $researchField,
bio: null,
phone: $phone !== '' ? $phone : null,
researchDirectionNames: $researchDirectionNames !== [] ? $researchDirectionNames : null,
);
if (count($items) >= $maxResults) {
@ -1517,15 +1673,23 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
return true;
}
if (preg_match('#/people/detail_new/\d+/?$#', $path)) {
return true;
}
if (preg_match('#/cn/facultydetails/[^/]+/[^/]+/?$#', $path)) {
return true;
}
if (preg_match('#/c\d+a\d+/page\.htm$#', $path)) {
return true;
}
if (preg_match('#/(?:szll|zjzjs)/[^/]+\.(?:htm|html)$#', $path)) {
if (preg_match('#/(?:szll|zjzjs)(?:/[^/]+)*/[^/]+\.(?:htm|html)$#', $path)) {
return true;
}
if (preg_match('#^(?:szll|zjzjs)/[^/]+\.(?:htm|html)$#', $path)) {
if (preg_match('#^(?:szll|zjzjs)(?:/[^/]+)*/[^/]+\.(?:htm|html)$#', $path)) {
return true;
}
@ -1794,8 +1958,8 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
$lead = is_array($item->extra['lead_author'] ?? null) ? $item->extra['lead_author'] : [];
$changed = false;
if (empty($lead['academic_title']) && preg_match('/<em>\s*([^<]+?)\s*<\/em>/u', $html, $titleMatch)) {
$title = CrawlAuthorParser::cleanText($titleMatch[1]);
if (empty($lead['academic_title'])) {
$title = $this->extractAcademicTitleFromProfileHtml($html);
if ($title !== null && $title !== '') {
$lead['academic_title'] = $title;
$changed = true;
@ -1803,7 +1967,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
}
if (empty($lead['college']) && empty($lead['affiliation'])) {
$dept = $this->parseLabeledField($html, '所属二级机构');
$dept = $this->parseLabeledField($html, '所属二级机构')
?? $this->parseLabeledField($html, '所在单位')
?? $this->parseLabeledField($html, '所在研究所');
if ($dept !== null && $dept !== '') {
$lead['affiliation'] = $dept;
$lead['college'] = $dept;
@ -1811,6 +1977,32 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
}
}
if (empty($lead['phone'] ?? null) && empty($item->extra['phone'] ?? null)) {
$phone = $this->extractPhoneFromProfileHtml($html);
if ($phone !== null && $phone !== '') {
$lead['phone'] = $phone;
$changed = true;
}
}
if (empty($lead['bio'] ?? null) && empty($item->extra['bio'] ?? null)) {
$bio = $this->extractBioFromProfileHtml($html);
if ($bio !== null && $bio !== '') {
$lead['bio'] = $bio;
$changed = true;
}
}
$existingDirections = $item->extra['research_direction_names'] ?? ($lead['research_direction_names'] ?? null);
if (! is_array($existingDirections) || $existingDirections === []) {
$directionText = $this->extractResearchDirectionTextFromProfileHtml($html);
$directionNames = $this->parseResearchDirectionNames($directionText ?? '');
if ($directionNames !== []) {
$lead['research_direction_names'] = $directionNames;
$changed = true;
}
}
if (! $changed) {
return $item;
}
@ -1823,6 +2015,15 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
if (! empty($lead['college'])) {
$extra['college_name'] = $lead['college'];
}
if (! empty($lead['phone'])) {
$extra['phone'] = $lead['phone'];
}
if (! empty($lead['bio'])) {
$extra['bio'] = $lead['bio'];
}
if (! empty($lead['research_direction_names']) && is_array($lead['research_direction_names'])) {
$extra['research_direction_names'] = $lead['research_direction_names'];
}
$authorsParsed = $item->authorsParsed;
if ($authorsParsed !== []) {
@ -1849,6 +2050,239 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
);
}
protected function extractAcademicTitleFromProfileHtml(string $html): ?string
{
if (preg_match('/<em>\s*([^<]+?)\s*<\/em>/u', $html, $titleMatch)) {
$title = CrawlAuthorParser::cleanText($titleMatch[1]);
if ($title !== null && $title !== '') {
return $title;
}
}
foreach (['职称', '职务'] as $label) {
$title = $this->parseLabeledField($html, $label);
if ($title !== null && $title !== '') {
return $title;
}
}
if (preg_match('#<div class="tit"[^>]*>.*?<span>([^<]+)</span>#su', $html, $match)) {
return CrawlAuthorParser::cleanText($match[1]);
}
return null;
}
protected function extractPhoneFromProfileHtml(string $html): ?string
{
$scoped = $this->profileContentHtml($html);
$patterns = [
'/办公电话[：:]\s*([0-9＋+\-—–()（）\s]{6,40})/u',
'/联系电话[：:]\s*([0-9＋+\-—–()（）\s]{6,40})/u',
'/details-tag">\s*联系电话\s*<\/span>\s*<span class="details-con">\s*([^<]+)/u',
'/电话[：:]\s*([0-9＋+\-—–()（）\s]{6,40})/u',
'/Tel(?:ephone)?[：:]\s*([0-9＋+\-—–()（）\s]{6,40})/iu',
'/Phone[：:]\s*([0-9＋+\-—–()（）\s]{6,40})/iu',
];
foreach ($patterns as $pattern) {
if (preg_match($pattern, $scoped, $match)) {
$phone = $this->normalizePhone($match[1]);
if ($phone !== '' && ! $this->isNoisePhone($phone)) {
return $phone;
}
}
}
return null;
}
protected function extractBioFromProfileHtml(string $html): ?string
{
$patterns = [
'#个人简介</(?:p|div|span)>\s*</div>\s*<div class="(?:txt|p|detail)"[^>]*>(.*?)</div>#su',
'#class="h3">\s*个人简介\s*</div>\s*<div class="p">(.*?)</div>#su',
'#class="name"><p>个人简介</p></div>\s*<div class="txt">(.*?)</div>#su',
'#info-subChannel"[^>]*>\s*<img[^>]*>\s*<span>\s*个人简介\s*</span>.*?<div class="info-ctx"[^>]*>(.*?)</div>\s*</div>#su',
'#个人简介[：:]\s*</span>\s*<div class="grjj"[^>]*>(.*?)</div>#su',
'#class="jj"><span>个人简介[：:]</span></div>\s*<div class="grjj"[^>]*>(.*?)</div>#su',
'#class="sz-jj[^"]*"[^>]*>.*?<div id="vsb_content[^"]*"[^>]*>(.*?)</div>#su',
'#主要学术成绩</[^>]+>.*?<div[^>]*>(.*?)</div>#su',
];
foreach ($patterns as $pattern) {
if (preg_match($pattern, $html, $match)) {
$bio = $this->htmlToPlain($match[1]);
$bio = CrawlAuthorParser::cleanText($bio);
if ($bio !== null && mb_strlen($bio) >= 20) {
return Str::limit($bio, 2000, '');
}
}
}
// 材料学院等：邮箱后的无标签长简介块
if (preg_match_all('#<div class="info[^"]*"[^>]*>(.*?)</div>#su', $html, $blocks)) {
foreach ($blocks[1] as $block) {
$plain = $this->htmlToPlain($block);
if (preg_match('/电子邮箱|通讯地址|所属二级机构|办公电话|联系电话/u', $plain)) {
continue;
}
$plain = CrawlAuthorParser::cleanText($plain);
if ($plain !== null && mb_strlen($plain) >= 40) {
return Str::limit($plain, 2000, '');
}
}
}
return null;
}
protected function extractResearchDirectionTextFromProfileHtml(string $html): ?string
{
$patterns = [
// 材料学院 panel：研究方向 + ul>li
'#研究方向\s*</div>.*?panel-body[^>]*>\s*<ul>(.*?)</ul>#su',
// 交大电气等：js-box-item name + txt
'#<div class="js-box-item[^"]*"[^>]*>\s*<div class="name">\s*研究兴趣\s*</div>\s*<div class="txt">(.*?)</div>#su',
'#<div class="js-box-item[^"]*"[^>]*>\s*研究兴趣\s*(.*?)</div>#su',
// 自动化/集成电路：tit 研究方向 + detail/txt
'#<div class="tit"[^>]*>\s*(?:<p>)?\s*研究方向\s*(?:</p>)?\s*</div>\s*<div class="detail"[^>]*>\s*<div class="txt">(.*?)</div>#su',
'#研究方向</(?:p|div)></div>\s*<div class="detail"[^>]*>\s*<div class="txt">(.*?)</div>#su',
// 南大前沿：研究领域
'#info-subChannel"[^>]*>\s*<img[^>]*>\s*<span>\s*研究领域\s*</span>.*?<div class="info-ctx"[^>]*>(.*?)</div>\s*</div>#su',
// 南大集成电路：研究方向 + div.yj
'#研究方向[：:]\s*</span>\s*<div class="yj">(.*?)</div>#su',
// 通用标签
'#研究(?:方向|领域|兴趣)[：:]\s*</?(?:strong|span|b|label)?[^>]*>\s*([^<]{2,500})#u',
'#研究(?:方向|领域|兴趣)[：:]\s*([^<\n]{2,500})#u',
];
foreach ($patterns as $pattern) {
if (! preg_match($pattern, $html, $match)) {
continue;
}
$chunk = (string) $match[1];
if (str_contains($chunk, '<li')) {
if (preg_match_all('#<li[^>]*>(.*?)</li>#su', $chunk, $lis)) {
$parts = [];
foreach ($lis[1] as $li) {
$text = CrawlAuthorParser::cleanText($this->htmlToPlain($li));
if ($text !== null && $text !== '') {
$parts[] = $text;
}
}
if ($parts !== []) {
return implode('、', $parts);
}
}
}
if (preg_match_all('#<p[^>]*>(.*?)</p>#su', $chunk, $paragraphs) && count($paragraphs[1]) > 1) {
$parts = [];
$numberedParts = [];
foreach ($paragraphs[1] as $paragraph) {
$text = CrawlAuthorParser::cleanText($this->htmlToPlain($paragraph));
if ($text === null || $text === '') {
continue;
}
$parts[] = $text;
if (preg_match('/^\d+[\.、．]/u', $text)) {
$numberedParts[] = preg_replace('/^\d+[\.、．]\s*/u', '', $text) ?? $text;
}
}
if ($numberedParts !== []) {
return implode('、', $numberedParts);
}
if ($parts !== []) {
return implode("\n", $parts);
}
}
$text = CrawlAuthorParser::cleanText($this->htmlToPlain($chunk));
if ($text !== null && $text !== '' && ! preg_match('/^(教育背景|工作经历|学术发表|项目资助)$/u', $text)) {
return $text;
}
}
return null;
}
/**
* @return list<string>
*/
protected function parseResearchDirectionNames(string $direction): array
{
$direction = trim($direction);
if ($direction === '') {
return [];
}
if (preg_match_all('/\d+[\.、．]\s*([^0-9\n]{2,80}?)(?=\s*\d+[\.、．]|[；;]|$)/u', $direction, $numbered)) {
$parts = array_values(array_unique(array_filter(array_map(
fn (string $part) => trim($part, " \t\n\r\0\x0B;；,，。"),
$numbered[1],
), fn (string $part) => $part !== '' && mb_strlen($part) <= 80)));
if (count($parts) >= 2) {
return $parts;
}
}
$parts = preg_split('/[、，,;；\/\|\r\n]+/u', $direction) ?: [];
return array_values(array_unique(array_filter(array_map(
function (string $part): string {
$part = trim($part);
$part = preg_replace('/^\d+[\.、．]\s*/u', '', $part) ?? $part;
return trim($part, " \t\n\r\0\x0B;；,，。.");
},
$parts,
), fn (string $part) => $part !== '' && mb_strlen($part) <= 80)));
}
protected function normalizePhone(string $phone): string
{
$phone = html_entity_decode($phone, ENT_QUOTES | ENT_HTML5, 'UTF-8');
$phone = trim(preg_replace('/\s+/u', ' ', $phone) ?? '');
$phone = trim($phone, " \t\n\r\0\x0B;；,，");
return $phone;
}
protected function isNoisePhone(string $phone): bool
{
$digits = preg_replace('/\D+/', '', $phone) ?? '';
if (strlen($digits) < 6) {
return true;
}
// 仅过滤明显占位号，保留 0 开头的座机号
return (bool) preg_match('/^(0{6,}|1{6,}|123456+)/', $digits);
}
protected function profileContentHtml(string $html): string
{
$scoped = preg_replace(
'#<(?:footer|div)[^>]*(?:class|id)=[\'"][^\'"]*(?:footer|foot-top|foot_nav|foot-logo|copyright)[^\'"]*[\'"][^>]*>.*$#isu',
'',
$html,
);
if (! is_string($scoped) || $scoped === '') {
$scoped = $html;
}
// 去掉页脚学院公共邮箱/电话残留
$scoped = preg_replace(
'#<(?:p|div)[^>]*>[^<]*(?:Copyright|版权所有|备案号)[\s\S]*$#iu',
'',
$scoped,
) ?? $scoped;
return $scoped;
}
/**
* @param list<string> $keywords
* @return list<CrawlItemDto>
@ -1937,6 +2371,7 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
/**
* @param list<string> $keywords
* @param list<string>|null $researchDirectionNames
*/
protected function makeFacultyItem(
string $externalKey,
@ -1950,17 +2385,22 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
?string $academicTitle,
string $platform,
?string $bio = null,
?string $phone = null,
?array $researchDirectionNames = null,
): CrawlItemDto {
$college = $affiliation;
$directionNames = array_values(array_filter($researchDirectionNames ?? []));
$lead = [
'name' => $name,
'email' => $email,
'phone' => $phone,
'affiliation' => $college,
'college' => $college,
'university_name' => $universityName,
'academic_title' => $academicTitle,
'bio' => $bio,
'profile_url' => $profileUrl,
'research_direction_names' => $directionNames !== [] ? $directionNames : null,
];
return new CrawlItemDto(
@ -1975,7 +2415,9 @@ class FacultyListHtmlAdapter implements CrawlerAdapterInterface
'academic_title' => $academicTitle,
'college_name' => $college,
'bio' => $bio,
'phone' => $phone,
'profile_url' => $profileUrl,
'research_direction_names' => $directionNames !== [] ? $directionNames : null,
'lead_author' => $lead,
'keyword' => implode(' ', $keywords),
],

269
app/Services/Crawl/CrawlImportService.php
Unescape View File

@ -29,6 +29,7 @@ class CrawlImportService
* papers_imported:int,
* teacher_leads_imported:int,
* teachers_imported:int,
* teachers_updated:int,
* news_imported:int
* }
*/
@ -48,13 +49,14 @@ class CrawlImportService
$query->whereIn('id', $itemIds);
}
$items = $query->get();
$items = $query->orderBy('id')->get();
$imported = 0;
$skipped = 0;
$failed = 0;
$papersImported = 0;
$teacherLeadsImported = 0;
$teachersImported = 0;
$teachersUpdated = 0;
$newsImported = 0;
DB::transaction(function () use (
@ -68,22 +70,26 @@ class CrawlImportService
&$papersImported,
&$teacherLeadsImported,
&$teachersImported,
&$teachersUpdated,
&$newsImported,
) {
foreach ($items as $item) {
try {
$id = match ($item->target_type) {
'paper' => $this->importPaper($job, $item),
$result = match ($item->target_type) {
'paper' => ['id' => $this->importPaper($job, $item), 'updated' => false],
'teacher_lead', 'teacher' => $this->importTeacher($job, $item, $teacherDefaults),
default => $this->importNews($job, $item, $newsDefaults),
default => ['id' => $this->importNews($job, $item, $newsDefaults), 'updated' => false],
};
$id = is_array($result) ? ($result['id'] ?? null) : $result;
$updated = is_array($result) ? (bool) ($result['updated'] ?? false) : false;
if ($id) {
$item->update(['status' => 'imported', 'target_id' => $id]);
$imported++;
match ($item->target_type) {
'paper' => $papersImported++,
'teacher_lead' => $teacherLeadsImported++,
'teacher' => $teachersImported++,
'teacher' => $updated ? $teachersUpdated++ : $teachersImported++,
default => $newsImported++,
};
} else {
@ -116,6 +122,7 @@ class CrawlImportService
'papers_imported' => $papersImported,
'teacher_leads_imported' => $teacherLeadsImported,
'teachers_imported' => $teachersImported,
'teachers_updated' => $teachersUpdated,
'news_imported' => $newsImported,
];
}
@ -172,8 +179,9 @@ class CrawlImportService
/**
* @param array{university_id?:int,department?:string,city?:string,research_direction_ids?:int[]} $defaults
* @return array{id:?int,updated:bool}
*/
protected function importTeacher(CrawlJob $job, CrawlJobItem $item, array $defaults = []): ?int
protected function importTeacher(CrawlJob $job, CrawlJobItem $item, array $defaults = []): array
{
$payload = $item->payload ?? [];
$lead = $payload['lead_author'] ?? null;
@ -188,18 +196,11 @@ class CrawlImportService
$name = trim((string) ($lead['name'] ?? ''));
if ($name === '') {
return null;
return ['id' => null, 'updated' => false];
}
$email = CrawlAuthorParser::normalizeEmail($lead['email'] ?? null);
if ($email && Teacher::query()->where('email', $email)->exists()) {
$item->update(['status' => 'duplicate']);
return null;
}
$phone = trim((string) ($lead['phone'] ?? $payload['phone'] ?? ''));
$academicTitle = trim((string) ($lead['academic_title'] ?? $payload['academic_title'] ?? ''));
$collegeName = trim((string) ($lead['college'] ?? $lead['affiliation'] ?? $payload['college_name'] ?? ''));
$defaultDepartment = trim((string) ($defaults['department'] ?? ''));
@ -222,10 +223,37 @@ class CrawlImportService
}
}
if ($this->teacherAlreadyExists($name, $email, $profileUrl, $universityId, $leadUniversityName, $collegeName)) {
$directionIds = $this->resolveTeacherDirectionIds($payload, $lead, $defaults);
$existing = $this->findExistingTeacher(
$name,
$email,
$profileUrl,
$universityId,
$leadUniversityName,
$collegeName,
);
if ($existing) {
$updated = $this->fillEmptyTeacherFields($existing, [
'email' => $email,
'phone' => $phone !== '' ? $phone : null,
'title' => $academicTitle !== '' ? $academicTitle : null,
'bio' => $bio !== '' ? $bio : null,
'university_id' => $universityId,
'department' => $collegeName !== '' ? $collegeName : null,
'city' => $city,
'profile_url' => $profileUrl !== '' ? $profileUrl : null,
'research_direction_ids' => $directionIds,
]);
if ($updated) {
return ['id' => $existing->id, 'updated' => true];
}
$item->update(['status' => 'duplicate']);
return null;
return ['id' => null, 'updated' => false];
}
$sourceId = $this->resolveTeacherSourceId($item->target_type);
@ -233,8 +261,6 @@ class CrawlImportService
throw new \RuntimeException('老师库字典未配置');
}
$statusId = null;
$remarkParts = [
match ($item->target_type) {
'teacher_lead' => '论文库入库',
@ -272,23 +298,11 @@ class CrawlImportService
'email' => $email,
'phone' => $phone !== '' ? $phone : null,
'source_dict_item_id' => $sourceId,
'status_dict_item_id' => $statusId,
'status_dict_item_id' => null,
'library_status' => $libraryStatus,
'remark' => implode('；', $remarkParts),
]);
$directionIds = [];
if (! empty($defaults['research_direction_ids']) && is_array($defaults['research_direction_ids'])) {
$directionIds = array_map('intval', $defaults['research_direction_ids']);
} else {
$directionNames = $payload['research_direction_names'] ?? $lead['research_direction_names'] ?? [];
if (is_string($directionNames)) {
$directionNames = preg_split('/[、，,;；\/]+/u', $directionNames) ?: [];
}
if (is_array($directionNames) && $directionNames !== []) {
$directionIds = app(ResearchDirectionResolver::class)->resolveIds([], $directionNames);
}
}
if ($directionIds !== []) {
$teacher->researchDirections()->sync($directionIds);
}
@ -304,7 +318,162 @@ class CrawlImportService
}
}
return $teacher->id;
return ['id' => $teacher->id, 'updated' => false];
}
/**
* @param array<string, mixed> $payload
* @param array<string, mixed> $lead
* @param array{university_id?:int,department?:string,city?:string,research_direction_ids?:int[]} $defaults
* @return list<int>
*/
protected function resolveTeacherDirectionIds(array $payload, array $lead, array $defaults): array
{
if (! empty($defaults['research_direction_ids']) && is_array($defaults['research_direction_ids'])) {
return array_values(array_unique(array_map('intval', $defaults['research_direction_ids'])));
}
$directionNames = $payload['research_direction_names'] ?? $lead['research_direction_names'] ?? [];
if (is_string($directionNames)) {
$directionNames = preg_split('/[、，,;；\/]+/u', $directionNames) ?: [];
}
if (! is_array($directionNames) || $directionNames === []) {
return [];
}
return app(ResearchDirectionResolver::class)->resolveIds([], $directionNames);
}
/**
* 仅补空字段：已有值不覆盖。
*
* @param array{
* email?:?string,
* phone?:?string,
* title?:?string,
* bio?:?string,
* university_id?:?int,
* department?:?string,
* city?:?string,
* profile_url?:?string,
* research_direction_ids?:list<int>
* } $incoming
*/
public function fillEmptyTeacherFields(Teacher $teacher, array $incoming): bool
{
$updates = [];
$changed = false;
$email = CrawlAuthorParser::normalizeEmail($incoming['email'] ?? null);
if ($this->isBlankTeacherValue($teacher->email) && $email) {
$emailTaken = Teacher::query()
->where('email', $email)
->where('id', '!=', $teacher->id)
->exists();
if (! $emailTaken) {
$updates['email'] = $email;
}
}
$phone = trim((string) ($incoming['phone'] ?? ''));
if ($this->isBlankTeacherValue($teacher->phone) && $phone !== '') {
$updates['phone'] = $phone;
}
$title = trim((string) ($incoming['title'] ?? ''));
if ($this->isBlankTeacherValue($teacher->title) && $title !== '') {
$updates['title'] = $title;
}
$bio = trim((string) ($incoming['bio'] ?? ''));
if ($this->isBlankTeacherValue($teacher->bio) && $bio !== '') {
$updates['bio'] = $bio;
}
$department = trim((string) ($incoming['department'] ?? ''));
if ($this->isBlankTeacherValue($teacher->department) && $department !== '') {
$updates['department'] = $department;
}
$city = trim((string) ($incoming['city'] ?? ''));
if ($this->isBlankTeacherValue($teacher->city) && $city !== '') {
$updates['city'] = $city;
}
$universityId = isset($incoming['university_id']) ? (int) $incoming['university_id'] : 0;
if (! $teacher->university_id && $universityId > 0) {
$updates['university_id'] = $universityId;
}
$profileUrl = trim((string) ($incoming['profile_url'] ?? ''));
if ($profileUrl !== '') {
$remark = (string) ($teacher->remark ?? '');
if ($remark === '' || ! str_contains($remark, $profileUrl)) {
$updates['remark'] = trim($remark.($remark !== '' ? '；' : '').'主页：'.$profileUrl, '；');
}
}
if ($updates !== []) {
$teacher->fill($updates);
$teacher->save();
$changed = true;
}
$directionIds = array_values(array_unique(array_filter(array_map(
'intval',
$incoming['research_direction_ids'] ?? [],
))));
if ($directionIds !== [] && $teacher->researchDirections()->count() === 0) {
$teacher->researchDirections()->sync($directionIds);
$changed = true;
}
return $changed;
}
protected function isBlankTeacherValue(mixed $value): bool
{
$text = trim((string) ($value ?? ''));
return $text === '' || $text === '待补充';
}
protected function findExistingTeacher(
string $name,
?string $email,
?string $profileUrl,
?int $universityId,
string $leadUniversityName,
?string $collegeName,
): ?Teacher {
if ($email) {
$byEmail = Teacher::query()->where('email', $email)->first();
if ($byEmail) {
return $byEmail;
}
}
if ($profileUrl !== null && $profileUrl !== '') {
$byProfile = Teacher::query()->where('remark', 'like', '%'.$profileUrl.'%')->first();
if ($byProfile) {
return $byProfile;
}
}
$dupQuery = Teacher::query()->where('name', $name);
if ($collegeName !== null && $collegeName !== '') {
$dupQuery->where('department', $collegeName);
}
if ($universityId) {
$dupQuery->where('university_id', $universityId);
} elseif ($leadUniversityName !== '') {
$dupQuery->where(function ($q) use ($leadUniversityName) {
$q->where('university_text', $leadUniversityName)
->orWhere('university_text', 'like', $leadUniversityName.'%');
});
}
return $dupQuery->first();
}
/**
@ -430,42 +599,6 @@ class CrawlImportService
}
}
protected function teacherAlreadyExists(
string $name,
?string $email,
?string $profileUrl,
?int $universityId,
string $leadUniversityName,
?string $collegeName,
): bool {
if ($email && Teacher::query()->where('email', $email)->exists()) {
return true;
}
if ($email) {
return false;
}
if ($profileUrl !== '') {
return Teacher::query()->where('remark', 'like', '%'.$profileUrl.'%')->exists();
}
$dupQuery = Teacher::query()->where('name', $name);
if ($collegeName !== '') {
$dupQuery->where('department', $collegeName);
}
if ($universityId) {
$dupQuery->where('university_id', $universityId);
} elseif ($leadUniversityName !== '') {
$dupQuery->where(function ($q) use ($leadUniversityName) {
$q->where('university_text', $leadUniversityName)
->orWhere('university_text', 'like', $leadUniversityName.'%');
});
}
return $dupQuery->exists();
}
protected function resolveUniversityId(string $name): ?int
{
$name = trim($name);

34
app/Services/Crawl/CrawlJobRunnerService.php
Unescape View File

@ -7,7 +7,6 @@ use App\Models\CrawlJobItem;
use App\Models\CrawlSource;
use App\Models\News;
use App\Models\Paper;
use App\Models\Teacher;
class CrawlJobRunnerService
{
@ -163,36 +162,8 @@ class CrawlJobRunnerService
protected function persistTeacherItem(CrawlJob $job, CrawlSource $source, CrawlItemDto $dto): bool
{
$lead = $dto->extra['lead_author'] ?? CrawlAuthorParser::leadAuthor($dto->authors, $dto->authorsParsed);
$email = is_array($lead) ? ($lead['email'] ?? null) : null;
$status = 'preview';
if ($email && Teacher::query()->where('email', $email)->exists()) {
$status = 'duplicate';
} elseif (is_array($lead)) {
$leadName = trim((string) ($lead['name'] ?? ''));
$leadUniversity = trim((string) ($lead['university_name'] ?? ''));
$profileUrl = trim((string) ($lead['profile_url'] ?? $dto->canonicalUrl ?? ''));
$collegeName = trim((string) ($lead['college'] ?? $lead['affiliation'] ?? $dto->extra['college_name'] ?? ''));
if ($leadName !== '' && ! $email) {
if ($profileUrl !== '' && Teacher::query()->where('remark', 'like', '%'.$profileUrl.'%')->exists()) {
$status = 'duplicate';
} else {
$dup = Teacher::query()->where('name', $leadName);
if ($collegeName !== '') {
$dup->where('department', $collegeName);
}
if ($leadUniversity !== '') {
$dup->where(function ($q) use ($leadUniversity) {
$q->where('university_text', $leadUniversity)
->orWhere('university_text', 'like', $leadUniversity.'%');
});
}
if ($dup->exists()) {
$status = 'duplicate';
}
}
}
}
// 已存在老师也先进入 preview，入库时仅补空字段；无空字段可补再标 duplicate
CrawlJobItem::query()->updateOrCreate(
[
'crawl_job_id' => $job->id,
@ -209,10 +180,11 @@ class CrawlJobRunnerService
'college_name' => $dto->extra['college_name'] ?? (is_array($lead) ? ($lead['college'] ?? $lead['affiliation'] ?? null) : null),
'profile_url' => $dto->extra['profile_url'] ?? $dto->canonicalUrl,
'phone' => $dto->extra['phone'] ?? (is_array($lead) ? ($lead['phone'] ?? null) : null),
'bio' => $dto->extra['bio'] ?? (is_array($lead) ? ($lead['bio'] ?? null) : null),
'research_direction_names' => $dto->extra['research_direction_names']
?? (is_array($lead) ? ($lead['research_direction_names'] ?? null) : null),
],
'status' => $status,
'status' => 'preview',
'target_type' => 'teacher',
'source_name' => $source->name,
'target_id' => null,

6
config/crawl.php
Unescape View File

@ -29,10 +29,10 @@ return [
],
'faculty' => [
/** 列表项无邮箱时，是否请求教师主页补全邮箱 */
/** 是否请求教师主页补全邮箱/电话/简介/研究方向 */
'profile_email_enrich_enabled' => (bool) env('FACULTY_PROFILE_EMAIL_ENRICH', true),
/** 单次任务最多补全主页数（其余仍入库，仅无邮箱） */
'profile_enrich_max' => (int) env('FACULTY_PROFILE_ENRICH_MAX', 32),
/** 单次任务最多补全主页数（缺邮箱/电话/简介/研究方向时访问详情页） */
'profile_enrich_max' => (int) env('FACULTY_PROFILE_ENRICH_MAX', 200),
'profile_http_timeout_seconds' => (int) env('FACULTY_PROFILE_HTTP_TIMEOUT', 10),
/** 并发请求教师主页数 */
'profile_enrich_pool_size' => (int) env('FACULTY_PROFILE_ENRICH_POOL', 8),

76
tests/Unit/CrawlImportTeacherFillEmptyTest.php
Unescape View File

@ -0,0 +1,76 @@
<?php
namespace Tests\Unit;
use App\Models\ResearchDirection;
use App\Models\Teacher;
use App\Services\Crawl\CrawlImportService;
use App\Support\TeacherLibraryStatus;
use Tests\TestCase;
class CrawlImportTeacherFillEmptyTest extends TestCase
{
public function test_fill_empty_teacher_fields_only_updates_blank_values(): void
{
$teacher = Teacher::query()->create([
'name' => '补空字段测试老师',
'university_id' => null,
'department' => '计算机学院',
'bio' => null,
'city' => '待补充',
'title' => '待补充',
'email' => null,
'phone' => null,
'library_status' => TeacherLibraryStatus::Active,
'remark' => '高校抓取入库',
]);
$direction = ResearchDirection::query()->firstOrCreate(
['name' => '补空字段测试方向'],
['sort' => 0, 'status' => 1, 'remark' => 'test'],
);
try {
$service = app(CrawlImportService::class);
$changed = $service->fillEmptyTeacherFields($teacher, [
'email' => 'fill-empty-test@sjtu.edu.cn',
'phone' => '021-12345678',
'title' => '教授',
'bio' => '长期从事操作系统研究。',
'city' => '上海',
'profile_url' => 'https://www.cs.sjtu.edu.cn/jiaoshiml/fill-empty-test.html',
'research_direction_ids' => [$direction->id],
]);
$this->assertTrue($changed);
$teacher->refresh();
$this->assertSame('fill-empty-test@sjtu.edu.cn', $teacher->email);
$this->assertSame('021-12345678', $teacher->phone);
$this->assertSame('教授', $teacher->title);
$this->assertSame('长期从事操作系统研究。', $teacher->bio);
$this->assertSame('上海', $teacher->city);
$this->assertStringContainsString('fill-empty-test.html', (string) $teacher->remark);
$this->assertSame([$direction->id], $teacher->researchDirections()->pluck('research_directions.id')->all());
// 已有值不应被覆盖
$changedAgain = $service->fillEmptyTeacherFields($teacher, [
'email' => 'other@sjtu.edu.cn',
'phone' => '021-87654321',
'title' => '副教授',
'bio' => '另一段简介',
'city' => '北京',
'research_direction_ids' => [$direction->id],
]);
$this->assertFalse($changedAgain);
$teacher->refresh();
$this->assertSame('fill-empty-test@sjtu.edu.cn', $teacher->email);
$this->assertSame('021-12345678', $teacher->phone);
$this->assertSame('教授', $teacher->title);
$this->assertSame('长期从事操作系统研究。', $teacher->bio);
$this->assertSame('上海', $teacher->city);
} finally {
$teacher->researchDirections()->detach();
$teacher->forceDelete();
}
}
}

186
tests/Unit/FacultyListHtmlAdapterTest.php
Unescape View File

@ -183,6 +183,17 @@ HTML;
$html = <<<'HTML'
<div class="people-name"><p>陈军</p><em>教授</em></div>
<div class="info jigou">所属二级机构：塑性成形技术与装备研究院</div>
<div class="info youxiangdizhi">电子邮箱：jun_chen@sjtu.edu.cn</div>
<div class="panel-head">
<div class="title">研究方向</div>
<div class="icon"><i class="iconfont icon-add-circle"></i></div>
</div>
<div class="panel-body">
<ul>
<li>轻质高强薄板塑性成形与数控渐进成形</li>
<li>塑性变形力学模型及其多物理场数值仿真</li>
</ul>
</div>
HTML;
$adapter = new FacultyListHtmlAdapter;
@ -208,6 +219,170 @@ HTML;
$this->assertSame('教授', $item->extra['lead_author']['academic_title']);
$this->assertSame('塑性成形技术与装备研究院', $item->extra['college_name']);
$this->assertSame(
['轻质高强薄板塑性成形与数控渐进成形', '塑性变形力学模型及其多物理场数值仿真'],
$item->extra['research_direction_names'],
);
}
public function test_apply_profile_metadata_from_see_detail_page(): void
{
$html = <<<'HTML'
<div class="js-info">
<p>办公电话：021-34204298</p>
<p>电子邮件：aiqian@sjtu.edu.cn</p>
</div>
<div class="js-box">
<div class="js-box-item">
<div class="name">研究兴趣</div>
<div class="txt"><p>电力系统元件建模, 电能质量, 分布式发电</p></div>
</div>
<div class="js-box-item">
<div class="name">教育背景</div>
<div class="txt"><p>1994.09 –1998.07清华大学</p></div>
</div>
</div>
<div class="footer"><p>邮箱：see@sjtu.edu.cn</p></div>
HTML;
$adapter = new FacultyListHtmlAdapter;
$method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem');
$method->setAccessible(true);
$item = $method->invoke(
$adapter,
new \App\Services\Crawl\CrawlItemDto(
externalId: 'faculty:see',
title: '艾芊',
canonicalUrl: 'https://see.sjtu.edu.cn/jiaoshiml/aiqian.html',
extra: ['lead_author' => ['name' => '艾芊']],
),
$html,
);
$this->assertSame('021-34204298', $item->extra['phone']);
$this->assertSame(['电力系统元件建模', '电能质量', '分布式发电'], $item->extra['research_direction_names']);
$emailMethod = new \ReflectionMethod($adapter, 'extractEmailFromProfileHtml');
$emailMethod->setAccessible(true);
$this->assertSame('aiqian@sjtu.edu.cn', $emailMethod->invoke($adapter, $html));
}
public function test_apply_profile_metadata_from_cs_and_frontier_pages(): void
{
$adapter = new FacultyListHtmlAdapter;
$method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem');
$method->setAccessible(true);
$csHtml = <<<'HTML'
<div class="dt"><p>邮箱：byzang@sjtu.edu.cn</p></div>
<div class="js-dt">
<div class="item item2">
<div class="name"><p>个人简介</p></div>
<div class="txt"><p>上海交通大学特聘教授，主要研究领域为操作系统、分布式系统与系统安全。</p></div>
</div>
</div>
HTML;
$csItem = $method->invoke(
$adapter,
new \App\Services\Crawl\CrawlItemDto(
externalId: 'faculty:cs',
title: '臧斌宇',
canonicalUrl: 'https://www.cs.sjtu.edu.cn/jiaoshiml/zangbinyu.html',
extra: ['lead_author' => ['name' => '臧斌宇']],
),
$csHtml,
);
$this->assertStringContainsString('操作系统', (string) $csItem->extra['bio']);
$frontierHtml = <<<'HTML'
<div class="person-contact">
<span class="details-tag">电子邮件</span>
<span class="details-con"><a href="mailto:baoming@nju.edu.cn">baoming@nju.edu.cn</a></span>
</div>
<div class="person-contact">
<span class="details-tag">联系电话</span>
<span class="details-con">0512-68768786</span>
</div>
<div class="info-item">
<div class="info-subChannel"><img src="/x.png"><span>研究领域</span></div>
<div class="info-ctx"><div class="wp_articlecontent"><p>微纳材料与微纳系统</p></div></div>
</div>
<div class="info-item">
<div class="info-subChannel"><img src="/x.png"><span>个人简介</span></div>
<div class="info-ctx"><div class="wp_articlecontent"><p>2006.09-2010.07 厦门大学，本科；长期从事微纳材料研究。</p></div></div>
</div>
HTML;
$frontierItem = $method->invoke(
$adapter,
new \App\Services\Crawl\CrawlItemDto(
externalId: 'faculty:frontier',
title: '王保明',
canonicalUrl: 'https://frontier.nju.edu.cn/85/ef/c59286a689647/page.htm',
extra: ['lead_author' => ['name' => '王保明']],
),
$frontierHtml,
);
$this->assertSame('0512-68768786', $frontierItem->extra['phone']);
$this->assertSame(['微纳材料与微纳系统'], $frontierItem->extra['research_direction_names']);
$this->assertStringContainsString('厦门大学', (string) $frontierItem->extra['bio']);
}
public function test_extracts_ra_teacher_list_with_research_directions(): void
{
$html = <<<'HTML'
<title>专职教师-南京大学机器人与自动化学院</title>
<ul class="teacher">
<li>
<a href="http://ra.nju.edu.cn/szll/zzjs/20250901/i335910.html" title="周克敏" target="_blank">
<div class="xm">周克敏</div>
<div>职称：<span>教授</span></div>
<div>研究方向：<span>鲁棒控制，多目标优化控制</span></div>
</a>
</li>
</ul>
HTML;
$adapter = new FacultyListHtmlAdapter;
$method = new \ReflectionMethod($adapter, 'extractFromHtml');
$method->setAccessible(true);
$items = $method->invoke($adapter, $html, [], 'https://ra.nju.edu.cn/szll/zzjs/index.html');
$this->assertCount(1, $items);
$this->assertSame('周克敏', $items[0]->title);
$this->assertSame('http://ra.nju.edu.cn/szll/zzjs/20250901/i335910.html', $items[0]->canonicalUrl);
$this->assertSame(['鲁棒控制', '多目标优化控制'], $items[0]->extra['research_direction_names']);
}
public function test_item_needs_profile_enrich_when_research_missing(): void
{
$adapter = new FacultyListHtmlAdapter;
$method = new \ReflectionMethod($adapter, 'itemNeedsProfileEnrich');
$method->setAccessible(true);
$item = new \App\Services\Crawl\CrawlItemDto(
externalId: 'faculty:x',
title: '张三',
canonicalUrl: 'https://see.sjtu.edu.cn/jiaoshiml/zhangsan.html',
extra: [
'lead_author' => [
'name' => '张三',
'email' => 'zhangsan@sjtu.edu.cn',
],
],
);
$this->assertTrue($method->invoke($adapter, $item));
}
public function test_resolve_profile_enrich_max_caps_large_batches(): void
{
$adapter = new FacultyListHtmlAdapter;
$method = new \ReflectionMethod($adapter, 'resolveProfileEnrichMax');
$method->setAccessible(true);
$this->assertSame(200, $method->invoke($adapter, [], 500));
$this->assertSame(10, $method->invoke($adapter, ['profile_enrich_max' => 10], 500));
$this->assertSame(0, $method->invoke($adapter, ['skip_profile_enrich' => true], 500));
}
public function test_extracts_sais_js_list_from_ajax_content(): void
@ -270,17 +445,6 @@ HTML;
$this->assertSame('https://www.cs.sjtu.edu.cn/jiaoshiml/chenhaibo.html', $items[1]->canonicalUrl);
}
public function test_resolve_profile_enrich_max_caps_large_batches(): void
{
$adapter = new FacultyListHtmlAdapter;
$method = new \ReflectionMethod($adapter, 'resolveProfileEnrichMax');
$method->setAccessible(true);
$this->assertSame(32, $method->invoke($adapter, [], 500));
$this->assertSame(10, $method->invoke($adapter, ['profile_enrich_max' => 10], 500));
$this->assertSame(0, $method->invoke($adapter, ['skip_profile_enrich' => true], 500));
}
public function test_response_body_from_pool_result_ignores_connection_exception(): void
{
$adapter = new FacultyListHtmlAdapter;

Write Preview
Loading…
Cancel
Save