|
|
<?php
|
|
|
|
|
|
namespace Tests\Unit;
|
|
|
|
|
|
use App\Services\Crawl\Adapters\FacultyListHtmlAdapter;
|
|
|
use Tests\TestCase;
|
|
|
|
|
|
class FacultyListHtmlAdapterTest extends TestCase
|
|
|
{
|
|
|
public function test_extracts_sjtu_college_teacher_list_without_email(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<div class="jssy-b">
|
|
|
<div class="title">电子信息与电气工程学院</div>
|
|
|
<div class="list">
|
|
|
<ul>
|
|
|
<li>
|
|
|
<a href="http://faculty.sjtu.edu.cn/bjshen/zh_CN/index.htm" target="_blank">
|
|
|
<div class="name">沈备军</div>
|
|
|
<p>所在单位:电子信息与电气工程学院</p>
|
|
|
<p>职称:副教授</p>
|
|
|
</a>
|
|
|
</li>
|
|
|
<li>
|
|
|
<a href="http://faculty.sjtu.edu.cn/xiabin/zh_CN/index.htm" target="_blank">
|
|
|
<div class="name">夏斌</div>
|
|
|
<p>所在单位:电子信息与电气工程学院</p>
|
|
|
<p>职称:教授</p>
|
|
|
</a>
|
|
|
</li>
|
|
|
</ul>
|
|
|
</div>
|
|
|
</div>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromHtml');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$items = $method->invoke(
|
|
|
$adapter,
|
|
|
$html,
|
|
|
[],
|
|
|
'https://faculty.sjtu.edu.cn/xyjs_list.jsp?urltype=tsites.CollegeTeacherList&wbtreeid=1001&st=0&id=1701&lang=zh_CN',
|
|
|
);
|
|
|
|
|
|
$this->assertCount(2, $items);
|
|
|
$this->assertSame('沈备军', $items[0]->title);
|
|
|
$this->assertSame('http://faculty.sjtu.edu.cn/bjshen/zh_CN/index.htm', $items[0]->canonicalUrl);
|
|
|
$this->assertSame('上海交通大学', $items[0]->schoolName);
|
|
|
$this->assertSame('faculty_html_tsites', $items[0]->extra['platform']);
|
|
|
$this->assertSame('副教授', $items[0]->extra['academic_title']);
|
|
|
$this->assertSame('电子信息与电气工程学院', $items[0]->extra['college_name']);
|
|
|
$this->assertSame('副教授', $items[0]->extra['lead_author']['academic_title']);
|
|
|
$this->assertSame('夏斌', $items[1]->title);
|
|
|
}
|
|
|
|
|
|
public function test_extracts_teacher_when_profile_href_is_empty(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<script>u_u11_pic.addimg("/__local/x.png","/shaohaibin/zh_CN/index.htm","邵海滨","1");</script>
|
|
|
<div class="list"><ul>
|
|
|
<li><a href="" target="_blank">
|
|
|
<div class="name">邵海滨</div>
|
|
|
<p>所在单位:电子信息与电气工程学院</p>
|
|
|
<p>职称:副研究员</p>
|
|
|
</a></li>
|
|
|
</ul></div>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromHtml');
|
|
|
$method->setAccessible(true);
|
|
|
$items = $method->invoke(
|
|
|
$adapter,
|
|
|
$html,
|
|
|
[],
|
|
|
'https://faculty.sjtu.edu.cn/xyjs_list.jsp?id=1701',
|
|
|
);
|
|
|
|
|
|
$this->assertCount(1, $items);
|
|
|
$this->assertSame('邵海滨', $items[0]->title);
|
|
|
$this->assertStringContainsString('shaohaibin', (string) $items[0]->canonicalUrl);
|
|
|
$this->assertSame('副研究员', $items[0]->extra['academic_title']);
|
|
|
$this->assertSame('电子信息与电气工程学院', $items[0]->extra['college_name']);
|
|
|
}
|
|
|
|
|
|
public function test_extracts_email_from_teacher_profile_html(): void
|
|
|
{
|
|
|
$html = '<li><strong>电子邮箱:</strong>bjshen@sjtu.edu.cn</li>';
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractEmailFromProfileHtml');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$email = $method->invoke($adapter, $html);
|
|
|
|
|
|
$this->assertSame('bjshen@sjtu.edu.cn', $email);
|
|
|
}
|
|
|
|
|
|
public function test_apply_email_to_item_updates_lead_author(): void
|
|
|
{
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'applyEmailToItem');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$item = $method->invoke(
|
|
|
$adapter,
|
|
|
new \App\Services\Crawl\CrawlItemDto(
|
|
|
externalId: 'faculty:test',
|
|
|
title: '沈备军',
|
|
|
canonicalUrl: 'http://faculty.sjtu.edu.cn/bjshen/zh_CN/index.htm',
|
|
|
extra: [
|
|
|
'lead_author' => [
|
|
|
'name' => '沈备军',
|
|
|
'email' => null,
|
|
|
'university_name' => '上海交通大学',
|
|
|
],
|
|
|
],
|
|
|
),
|
|
|
'bjshen@sjtu.edu.cn',
|
|
|
);
|
|
|
|
|
|
$this->assertSame('bjshen@sjtu.edu.cn', $item->extra['lead_author']['email']);
|
|
|
}
|
|
|
|
|
|
public function test_detects_total_pages_and_builds_pagenum_url(): void
|
|
|
{
|
|
|
$html = '<a href="?totalpage=20&PAGENUM=2&urltype=tsites.CollegeTeacherList&id=1701">下页</a>';
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
|
|
|
$detect = new \ReflectionMethod($adapter, 'detectTotalPages');
|
|
|
$detect->setAccessible(true);
|
|
|
$this->assertSame(20, $detect->invoke($adapter, $html));
|
|
|
|
|
|
$build = new \ReflectionMethod($adapter, 'buildPageUrl');
|
|
|
$build->setAccessible(true);
|
|
|
$url = $build->invoke(
|
|
|
$adapter,
|
|
|
'https://faculty.sjtu.edu.cn/xyjs_list.jsp?urltype=tsites.CollegeTeacherList&wbtreeid=1001&st=0&id=1701&lang=zh_CN',
|
|
|
3,
|
|
|
$html,
|
|
|
);
|
|
|
$this->assertStringContainsString('PAGENUM=3', $url);
|
|
|
$this->assertStringContainsString('totalpage=20', $url);
|
|
|
}
|
|
|
|
|
|
public function test_extracts_smse_staff_panel_list(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<title>教师名录 - 上海交通大学材料科学与工程学院</title>
|
|
|
<div class="panel-head">
|
|
|
<div class="title">塑性成形技术与装备研究院</div>
|
|
|
</div>
|
|
|
<div class="panel-body">
|
|
|
<div class="staff-list">
|
|
|
<a href="/people/detail_new/20092" class="staff-item">陈军</a>
|
|
|
<a href="/people/detail_new/20111" class="staff-item">韩先洪</a>
|
|
|
</div>
|
|
|
</div>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromHtml');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$items = $method->invoke(
|
|
|
$adapter,
|
|
|
$html,
|
|
|
[],
|
|
|
'https://smse.sjtu.edu.cn/people/staff_new/department',
|
|
|
);
|
|
|
|
|
|
$this->assertCount(2, $items);
|
|
|
$this->assertSame('陈军', $items[0]->title);
|
|
|
$this->assertSame('https://smse.sjtu.edu.cn/people/detail_new/20092', $items[0]->canonicalUrl);
|
|
|
$this->assertSame('上海交通大学', $items[0]->schoolName);
|
|
|
$this->assertSame('faculty_html_smse', $items[0]->extra['platform']);
|
|
|
$this->assertSame('塑性成形技术与装备研究院', $items[0]->extra['college_name']);
|
|
|
}
|
|
|
|
|
|
public function test_apply_profile_metadata_from_smse_detail_page(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<div class="people-name"><p>陈军</p><em>教授</em></div>
|
|
|
<div class="info jigou">所属二级机构:塑性成形技术与装备研究院</div>
|
|
|
<div class="info youxiangdizhi">电子邮箱:jun_chen@sjtu.edu.cn</div>
|
|
|
<div class="panel-head">
|
|
|
<div class="title">研究方向</div>
|
|
|
<div class="icon"><i class="iconfont icon-add-circle"></i></div>
|
|
|
</div>
|
|
|
<div class="panel-body">
|
|
|
<ul>
|
|
|
<li>轻质高强薄板塑性成形与数控渐进成形</li>
|
|
|
<li>塑性变形力学模型及其多物理场数值仿真</li>
|
|
|
</ul>
|
|
|
</div>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$item = $method->invoke(
|
|
|
$adapter,
|
|
|
new \App\Services\Crawl\CrawlItemDto(
|
|
|
externalId: 'faculty:test',
|
|
|
title: '陈军',
|
|
|
canonicalUrl: 'https://smse.sjtu.edu.cn/people/detail_new/20092',
|
|
|
extra: [
|
|
|
'lead_author' => [
|
|
|
'name' => '陈军',
|
|
|
'email' => null,
|
|
|
'university_name' => '上海交通大学',
|
|
|
],
|
|
|
],
|
|
|
),
|
|
|
$html,
|
|
|
);
|
|
|
|
|
|
$this->assertSame('教授', $item->extra['lead_author']['academic_title']);
|
|
|
$this->assertSame('塑性成形技术与装备研究院', $item->extra['college_name']);
|
|
|
$this->assertSame(
|
|
|
['轻质高强薄板塑性成形与数控渐进成形', '塑性变形力学模型及其多物理场数值仿真'],
|
|
|
$item->extra['research_direction_names'],
|
|
|
);
|
|
|
}
|
|
|
|
|
|
public function test_apply_profile_metadata_from_see_detail_page(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<div class="js-info">
|
|
|
<p>办公电话:021-34204298</p>
|
|
|
<p>电子邮件:aiqian@sjtu.edu.cn</p>
|
|
|
</div>
|
|
|
<div class="js-box">
|
|
|
<div class="js-box-item">
|
|
|
<div class="name">研究兴趣</div>
|
|
|
<div class="txt"><p>电力系统元件建模, 电能质量, 分布式发电</p></div>
|
|
|
</div>
|
|
|
<div class="js-box-item">
|
|
|
<div class="name">教育背景</div>
|
|
|
<div class="txt"><p>1994.09 –1998.07清华大学</p></div>
|
|
|
</div>
|
|
|
</div>
|
|
|
<div class="footer"><p>邮箱:see@sjtu.edu.cn</p></div>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem');
|
|
|
$method->setAccessible(true);
|
|
|
$item = $method->invoke(
|
|
|
$adapter,
|
|
|
new \App\Services\Crawl\CrawlItemDto(
|
|
|
externalId: 'faculty:see',
|
|
|
title: '艾芊',
|
|
|
canonicalUrl: 'https://see.sjtu.edu.cn/jiaoshiml/aiqian.html',
|
|
|
extra: ['lead_author' => ['name' => '艾芊']],
|
|
|
),
|
|
|
$html,
|
|
|
);
|
|
|
|
|
|
$this->assertSame('021-34204298', $item->extra['phone']);
|
|
|
$this->assertSame(['电力系统元件建模', '电能质量', '分布式发电'], $item->extra['research_direction_names']);
|
|
|
|
|
|
$emailMethod = new \ReflectionMethod($adapter, 'extractEmailFromProfileHtml');
|
|
|
$emailMethod->setAccessible(true);
|
|
|
$this->assertSame('aiqian@sjtu.edu.cn', $emailMethod->invoke($adapter, $html));
|
|
|
}
|
|
|
|
|
|
public function test_apply_profile_metadata_from_see_tit_span_not_nav_em(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<a href="/" title="电气首页" class="name "><em>电气首页</em></a>
|
|
|
<h2><a href="/" title="首页">电气首页</a></h2>
|
|
|
<div class="tit">
|
|
|
<p>王承民</p>
|
|
|
<span>教授</span>
|
|
|
</div>
|
|
|
<div class="js-info">
|
|
|
<p>办公电话:021-12345678</p>
|
|
|
<p>电子邮件:wcm@sjtu.edu.cn</p>
|
|
|
</div>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem');
|
|
|
$method->setAccessible(true);
|
|
|
$item = $method->invoke(
|
|
|
$adapter,
|
|
|
new \App\Services\Crawl\CrawlItemDto(
|
|
|
externalId: 'faculty:see-wcm',
|
|
|
title: '王承民',
|
|
|
canonicalUrl: 'https://see.sjtu.edu.cn/jiaoshiml/wangchengmin.html',
|
|
|
extra: [
|
|
|
'lead_author' => [
|
|
|
'name' => '王承民',
|
|
|
// 模拟上次误抓导航文案,详情页应覆盖
|
|
|
'academic_title' => '电气首页',
|
|
|
],
|
|
|
],
|
|
|
),
|
|
|
$html,
|
|
|
);
|
|
|
|
|
|
$this->assertSame('教授', $item->extra['academic_title']);
|
|
|
$this->assertSame('教授', $item->extra['lead_author']['academic_title']);
|
|
|
}
|
|
|
|
|
|
public function test_apply_profile_metadata_from_cs_and_frontier_pages(): void
|
|
|
{
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$csHtml = <<<'HTML'
|
|
|
<div class="js-info">
|
|
|
<div class="txt">
|
|
|
<div class="name">张晓凡</div>
|
|
|
<div class="zw">长聘教轨副教授</div>
|
|
|
<div class="dt">
|
|
|
<p>邮箱:xiaofan.zhang@sjtu.edu.cn</p>
|
|
|
<p>地址:剑川路930号A栋322B</p>
|
|
|
<p>所在研究所:清源研究院</p>
|
|
|
<p>个人主页:<a href="https://zhangxiaofan101.github.io" target="_blank">https://zhangxiaofan101.github.io</a></p>
|
|
|
</div>
|
|
|
</div>
|
|
|
</div>
|
|
|
<div class="js-dt">
|
|
|
<div class="item item2">
|
|
|
<div class="name"><p>个人简介</p></div>
|
|
|
<div class="txt"><p><span>上海交通大学计算机学院清源研究院长聘教轨副教授,博士生导师。</span><span>研究领域:医学图像、文本分析,多模态决策。</span></p></div>
|
|
|
</div>
|
|
|
<div class="item">
|
|
|
<div class="name"><p>教育背景</p></div>
|
|
|
<div class="txt"><p>北京航空航天大学学士,美国北卡罗莱纳大学夏洛特分校博士。</p></div>
|
|
|
</div>
|
|
|
</div>
|
|
|
HTML;
|
|
|
$csItem = $method->invoke(
|
|
|
$adapter,
|
|
|
new \App\Services\Crawl\CrawlItemDto(
|
|
|
externalId: 'faculty:cs',
|
|
|
title: '张晓凡',
|
|
|
canonicalUrl: 'https://www.cs.sjtu.edu.cn/jiaoshiml/zhangxiaofan.html',
|
|
|
extra: ['lead_author' => ['name' => '张晓凡']],
|
|
|
),
|
|
|
$csHtml,
|
|
|
);
|
|
|
$this->assertSame('长聘教轨副教授', $csItem->extra['academic_title']);
|
|
|
$this->assertSame('清源研究院', $csItem->extra['college_name']);
|
|
|
$this->assertStringContainsString('医学图像', (string) $csItem->extra['bio']);
|
|
|
$this->assertStringNotContainsString('教育背景', (string) $csItem->extra['bio']);
|
|
|
$this->assertStringNotContainsString('北京航空航天大学学士', (string) $csItem->extra['bio']);
|
|
|
|
|
|
$emailMethod = new \ReflectionMethod($adapter, 'extractEmailFromProfileHtml');
|
|
|
$emailMethod->setAccessible(true);
|
|
|
$this->assertSame('xiaofan.zhang@sjtu.edu.cn', $emailMethod->invoke($adapter, $csHtml));
|
|
|
|
|
|
$frontierHtml = <<<'HTML'
|
|
|
<div class="person-contact">
|
|
|
<span class="details-tag">电子邮件</span>
|
|
|
<span class="details-con"><a href="mailto:baoming@nju.edu.cn">baoming@nju.edu.cn</a></span>
|
|
|
</div>
|
|
|
<div class="person-contact">
|
|
|
<span class="details-tag">联系电话</span>
|
|
|
<span class="details-con">0512-68768786</span>
|
|
|
</div>
|
|
|
<div class="info-item">
|
|
|
<div class="info-subChannel"><img src="/x.png"><span>研究领域</span></div>
|
|
|
<div class="info-ctx"><div class="wp_articlecontent"><p>微纳材料与微纳系统</p></div></div>
|
|
|
</div>
|
|
|
<div class="info-item">
|
|
|
<div class="info-subChannel"><img src="/x.png"><span>个人简介</span></div>
|
|
|
<div class="info-ctx"><div class="wp_articlecontent"><p>2006.09-2010.07 厦门大学,本科;长期从事微纳材料研究。</p></div></div>
|
|
|
</div>
|
|
|
HTML;
|
|
|
$frontierItem = $method->invoke(
|
|
|
$adapter,
|
|
|
new \App\Services\Crawl\CrawlItemDto(
|
|
|
externalId: 'faculty:frontier',
|
|
|
title: '王保明',
|
|
|
canonicalUrl: 'https://frontier.nju.edu.cn/85/ef/c59286a689647/page.htm',
|
|
|
extra: ['lead_author' => ['name' => '王保明']],
|
|
|
),
|
|
|
$frontierHtml,
|
|
|
);
|
|
|
$this->assertSame('0512-68768786', $frontierItem->extra['phone']);
|
|
|
$this->assertSame(['微纳材料与微纳系统'], $frontierItem->extra['research_direction_names']);
|
|
|
$this->assertStringContainsString('厦门大学', (string) $frontierItem->extra['bio']);
|
|
|
}
|
|
|
|
|
|
public function test_extracts_ra_teacher_list_with_research_directions(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<title>专职教师-南京大学机器人与自动化学院</title>
|
|
|
<ul class="teacher">
|
|
|
<li>
|
|
|
<a href="http://ra.nju.edu.cn/szll/zzjs/20250901/i335910.html" title="周克敏" target="_blank">
|
|
|
<div class="xm">周克敏</div>
|
|
|
<div>职称:<span>教授</span></div>
|
|
|
<div>研究方向:<span>鲁棒控制,多目标优化控制</span></div>
|
|
|
</a>
|
|
|
</li>
|
|
|
</ul>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromHtml');
|
|
|
$method->setAccessible(true);
|
|
|
$items = $method->invoke($adapter, $html, [], 'https://ra.nju.edu.cn/szll/zzjs/index.html');
|
|
|
|
|
|
$this->assertCount(1, $items);
|
|
|
$this->assertSame('周克敏', $items[0]->title);
|
|
|
$this->assertSame('http://ra.nju.edu.cn/szll/zzjs/20250901/i335910.html', $items[0]->canonicalUrl);
|
|
|
$this->assertSame(['鲁棒控制', '多目标优化控制'], $items[0]->extra['research_direction_names']);
|
|
|
}
|
|
|
|
|
|
public function test_item_needs_profile_enrich_when_research_missing(): void
|
|
|
{
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'itemNeedsProfileEnrich');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$item = new \App\Services\Crawl\CrawlItemDto(
|
|
|
externalId: 'faculty:x',
|
|
|
title: '张三',
|
|
|
canonicalUrl: 'https://see.sjtu.edu.cn/jiaoshiml/zhangsan.html',
|
|
|
extra: [
|
|
|
'lead_author' => [
|
|
|
'name' => '张三',
|
|
|
'email' => 'zhangsan@sjtu.edu.cn',
|
|
|
],
|
|
|
],
|
|
|
);
|
|
|
|
|
|
$this->assertTrue($method->invoke($adapter, $item));
|
|
|
}
|
|
|
|
|
|
public function test_resolve_profile_enrich_max_caps_large_batches(): void
|
|
|
{
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'resolveProfileEnrichMax');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$this->assertSame(500, $method->invoke($adapter, [], 500));
|
|
|
$this->assertSame(10, $method->invoke($adapter, ['profile_enrich_max' => 10], 500));
|
|
|
$this->assertSame(300, $method->invoke($adapter, ['max_results' => 300], 300));
|
|
|
$this->assertSame(0, $method->invoke($adapter, ['skip_profile_enrich' => true], 500));
|
|
|
}
|
|
|
|
|
|
public function test_extracts_sais_js_list_from_ajax_content(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<title>教师名录-上海交通大学自动化与感知学院</title>
|
|
|
<div class="js-list">
|
|
|
<li><a href="https://sais.sjtu.edu.cn/faculty/baiyang.html" class="name">白洋</a></li>
|
|
|
<li><a href="https://sais.sjtu.edu.cn/faculty/chenxin.html" class="name">陈新</a></li>
|
|
|
</div>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromAjaxTeacherContent');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$items = $method->invoke(
|
|
|
$adapter,
|
|
|
$html,
|
|
|
[],
|
|
|
'https://sais.sjtu.edu.cn/faculty.html',
|
|
|
'faculty',
|
|
|
);
|
|
|
|
|
|
$this->assertCount(2, $items);
|
|
|
$this->assertSame('白洋', $items[0]->title);
|
|
|
$this->assertSame('https://sais.sjtu.edu.cn/faculty/baiyang.html', $items[0]->canonicalUrl);
|
|
|
$this->assertSame('faculty_html_ajax', $items[0]->extra['platform']);
|
|
|
$this->assertSame('上海交通大学', $items[0]->schoolName);
|
|
|
}
|
|
|
|
|
|
public function test_extracts_cs_rc_item_teacher_list(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<title>教师名录-上海交通大学计算机学院(网络空间安全学院、密码学院)</title>
|
|
|
<div class="rc-item">
|
|
|
<div class="tit"><div class="name">并行与分布式系统研究所</div></div>
|
|
|
<div class="dt">
|
|
|
<p>所长:<a href="https://www.cs.sjtu.edu.cn/jiaoshiml/zangbinyu.html" target="_blank">臧斌宇</a></p>
|
|
|
<p><a href="https://www.cs.sjtu.edu.cn/jiaoshiml/chenhaibo.html" target="_blank">陈海波</a></p>
|
|
|
</div>
|
|
|
</div>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromAjaxTeacherContent');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$items = $method->invoke(
|
|
|
$adapter,
|
|
|
$html,
|
|
|
[],
|
|
|
'https://www.cs.sjtu.edu.cn/jiaoshiml.html',
|
|
|
'jiaoshiml',
|
|
|
);
|
|
|
|
|
|
$this->assertCount(2, $items);
|
|
|
$this->assertSame('臧斌宇', $items[0]->title);
|
|
|
$this->assertSame('并行与分布式系统研究所', $items[0]->extra['college_name']);
|
|
|
$this->assertSame('https://www.cs.sjtu.edu.cn/jiaoshiml/chenhaibo.html', $items[1]->canonicalUrl);
|
|
|
}
|
|
|
|
|
|
public function test_response_body_from_pool_result_ignores_connection_exception(): void
|
|
|
{
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'responseBodyFromPoolResult');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$this->assertNull($method->invoke($adapter, new \GuzzleHttp\Exception\ConnectException(
|
|
|
'Connection timed out',
|
|
|
new \GuzzleHttp\Psr7\Request('GET', 'https://faculty.sjtu.edu.cn/test'),
|
|
|
)));
|
|
|
$this->assertNull($method->invoke($adapter, null));
|
|
|
}
|
|
|
|
|
|
public function test_parses_icisee_ajax_teacher_config_without_cat_id(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<script>
|
|
|
$.ajax({
|
|
|
url: '/active/ajax_teacher_list.html',
|
|
|
type: 'post',
|
|
|
data: {page:page, cat_code:'jiaoshiml', yjszxfl:global_yjszxfl, name:global_name, zm:global_zm},
|
|
|
});
|
|
|
</script>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'parseAjaxTeacherConfig');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$config = $method->invoke($adapter, $html, 'https://icisee.sjtu.edu.cn/jiaoshiml.html');
|
|
|
|
|
|
$this->assertSame('simple', $config['variant']);
|
|
|
$this->assertNull($config['cat_id']);
|
|
|
$this->assertSame('jiaoshiml', $config['cat_code']);
|
|
|
$this->assertSame('https://icisee.sjtu.edu.cn/active/ajax_teacher_list.html', $config['api_url']);
|
|
|
$this->assertTrue($config['uses_page']);
|
|
|
}
|
|
|
|
|
|
public function test_extracts_icisee_card_style_teacher_list(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<title>教师名录-上海交通大学集成电路学院(信息与电子工程学院)</title>
|
|
|
<a href="/jiaoshiml/caixinghan.html" target="_blank">
|
|
|
<div class="imgk"><img src="/upload/x.png" alt=""></div>
|
|
|
<div class="name">蔡星汉<span>教授</span><p class="line-2">微纳全重党支部书记</p></div>
|
|
|
</a>
|
|
|
<a href="/jiaoshiml/zhangsan.html" target="_blank">
|
|
|
<div class="imgk"><img src="/upload/y.png" alt=""></div>
|
|
|
<div class="name">张三<span>副教授</span></div>
|
|
|
</a>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromAjaxTeacherContent');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$items = $method->invoke(
|
|
|
$adapter,
|
|
|
$html,
|
|
|
[],
|
|
|
'https://icisee.sjtu.edu.cn/jiaoshiml.html',
|
|
|
'jiaoshiml',
|
|
|
);
|
|
|
|
|
|
$this->assertCount(2, $items);
|
|
|
$this->assertSame('蔡星汉', $items[0]->title);
|
|
|
$this->assertSame('教授', $items[0]->extra['academic_title']);
|
|
|
$this->assertSame('https://icisee.sjtu.edu.cn/jiaoshiml/caixinghan.html', $items[0]->canonicalUrl);
|
|
|
$this->assertSame('张三', $items[1]->title);
|
|
|
$this->assertSame('副教授', $items[1]->extra['academic_title']);
|
|
|
$this->assertSame('faculty_html_ajax', $items[0]->extra['platform']);
|
|
|
}
|
|
|
|
|
|
public function test_detects_nju_teacher_home_page(): void
|
|
|
{
|
|
|
$html = '<body class="list faculty"><script src="/js/faculty.js"></script></body>';
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'isNjuTeacherHomePage');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$this->assertTrue($method->invoke($adapter, $html));
|
|
|
$this->assertFalse($method->invoke($adapter, '<body class="list"><script src="/js/list.js"></script></body>'));
|
|
|
}
|
|
|
|
|
|
public function test_builds_nju_teacher_home_conditions_for_all_faculty(): void
|
|
|
{
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'buildNjuTeacherHomeConditions');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$conditions = $method->invoke($adapter, null, null);
|
|
|
|
|
|
$this->assertCount(1, $conditions);
|
|
|
$this->assertSame('published', $conditions[0]['field']);
|
|
|
}
|
|
|
|
|
|
public function test_builds_nju_teacher_home_conditions_for_professor_category(): void
|
|
|
{
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'buildNjuTeacherHomeConditions');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$conditions = $method->invoke($adapter, '教授', null);
|
|
|
|
|
|
$this->assertCount(2, $conditions);
|
|
|
$this->assertSame('exField2', $conditions[1]['field']);
|
|
|
$this->assertSame('教授', $conditions[1]['value']);
|
|
|
}
|
|
|
|
|
|
public function test_parses_nju_site_id_from_html(): void
|
|
|
{
|
|
|
$html = '<script src="/_js/jquery.min.js" sudy-wp-siteId="786"></script>';
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'parseNjuSiteId');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$this->assertSame(786, $method->invoke($adapter, $html));
|
|
|
}
|
|
|
|
|
|
public function test_infers_college_from_meta_description(): void
|
|
|
{
|
|
|
$html = '<title>师资力量</title><meta name="description" content="南京大学智能科学与技术学院" >';
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'inferCollegeFromPageTitle');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$this->assertSame('智能科学与技术学院', $method->invoke($adapter, $html));
|
|
|
}
|
|
|
|
|
|
public function test_extracts_sudy_news_faculty_list(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<title>师资力量-南京大学前沿科学学院</title>
|
|
|
<li class="wp_sublist sublist-1">
|
|
|
<h3 class="sublist_title"><span class="subcolumn-name">功能材料与智能制造研究院</span></h3>
|
|
|
<ul class="news_list list2">
|
|
|
<li class="news n1 clearfix">
|
|
|
<div class="news_title"><a href='/85/ef/c59286a689647/page.htm' title='王保明'>王保明</a></div>
|
|
|
</li>
|
|
|
</ul>
|
|
|
</li>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromSudyNewsFacultyList');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$items = $method->invoke($adapter, $html, [], 'https://frontier.nju.edu.cn/zrjs/list.htm');
|
|
|
|
|
|
$this->assertCount(1, $items);
|
|
|
$this->assertSame('王保明', $items[0]->title);
|
|
|
$this->assertSame('功能材料与智能制造研究院', $items[0]->extra['college_name']);
|
|
|
$this->assertSame('faculty_html_sudy_news', $items[0]->extra['platform']);
|
|
|
}
|
|
|
|
|
|
public function test_extracts_ic_nju_titles_from_news_title1_and_comments(): void
|
|
|
{
|
|
|
$qyml = <<<'HTML'
|
|
|
<title>全院名录-南京大学集成电路学院</title>
|
|
|
<ul class="news_list list2">
|
|
|
<li class="news n1 clearfix">
|
|
|
<span class="news_title"><a href='/af/8a/c56672a634762/page.htm' target='_blank' title='王欣然'>王欣然</a><span class="news_title1">教授,博导</span></span>
|
|
|
</li>
|
|
|
</ul>
|
|
|
HTML;
|
|
|
$overview = <<<'HTML'
|
|
|
<title>师资队伍-南京大学集成电路学院</title>
|
|
|
<ul class="news_list list2">
|
|
|
<li class="news n1 clearfix">
|
|
|
<!--<span class="news_title"><a href='/af/8a/c56672a634762/page.htm' target='_blank' title='王欣然'>王欣然</a><span class="news_title1">教授,博导</span></span>-->
|
|
|
<span class="news_title1"><a href='/af/8a/c56672a634762/page.htm' target='_blank' title='王欣然'>王欣然</a></span>
|
|
|
</li>
|
|
|
</ul>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromSudyNewsFacultyList');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$qymlItems = $method->invoke($adapter, $qyml, [], 'https://ic.nju.edu.cn/qyml/list.htm');
|
|
|
$this->assertSame('王欣然', $qymlItems[0]->title);
|
|
|
$this->assertSame('教授,博导', $qymlItems[0]->extra['academic_title']);
|
|
|
|
|
|
$overviewItems = $method->invoke($adapter, $overview, [], 'https://ic.nju.edu.cn/56606/list.htm');
|
|
|
$this->assertSame('王欣然', $overviewItems[0]->title);
|
|
|
$this->assertSame('教授,博导', $overviewItems[0]->extra['academic_title']);
|
|
|
|
|
|
$profileHtml = '<div class="zc"><span>职称:</span>教授,博导</div>';
|
|
|
$titleMethod = new \ReflectionMethod($adapter, 'extractAcademicTitleFromProfileHtml');
|
|
|
$titleMethod->setAccessible(true);
|
|
|
$this->assertSame('教授,博导', $titleMethod->invoke($adapter, $profileHtml));
|
|
|
}
|
|
|
|
|
|
public function test_extracts_ra_teacher_cards(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<title>专职教师-南京大学机器人与自动化学院</title>
|
|
|
<ul class="teacher">
|
|
|
<li>
|
|
|
<a href="http://ra.nju.edu.cn/szll/zzjs/20250901/i335910.html" title="周克敏">
|
|
|
<div class="data"><div class="name"><div class="xm">周克敏</div></div>
|
|
|
<div class="research pro">职称:<span>教授</span></div>
|
|
|
<div class="research">研究方向:<span>鲁棒控制</span></div></div>
|
|
|
</a>
|
|
|
</li>
|
|
|
</ul>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromRaTeacherList');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$items = $method->invoke($adapter, $html, [], 'https://ra.nju.edu.cn/szll/zzjs/index.html');
|
|
|
|
|
|
$this->assertCount(1, $items);
|
|
|
$this->assertSame('周克敏', $items[0]->title);
|
|
|
$this->assertSame('教授', $items[0]->extra['academic_title']);
|
|
|
$this->assertSame('faculty_html_ra', $items[0]->extra['platform']);
|
|
|
}
|
|
|
|
|
|
public function test_extracts_vsb_faculty_table(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<title>专兼职教师-南京大学智能软件与工程学院</title>
|
|
|
<div class="zjzjs"><p><strong><span>教授</span></strong></p>
|
|
|
<table><tr><td><a href="zjzjs/yangkun.htm"><span>杨鲲</span></a></td><td><span>陶先平</span></td></tr></table>
|
|
|
<p><strong><span>副教授</span></strong></p>
|
|
|
<table><tr><td><a href="zjzjs/shaodong.htm"><span>邵栋</span></a></td></tr></table>
|
|
|
</div>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'extractFromVsbFacultyTable');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$items = $method->invoke($adapter, $html, [], 'https://ise.nju.edu.cn/szll/zjzjs.htm');
|
|
|
|
|
|
$this->assertCount(3, $items);
|
|
|
$names = array_map(fn ($item) => $item->title, $items);
|
|
|
$this->assertContains('杨鲲', $names);
|
|
|
$this->assertContains('陶先平', $names);
|
|
|
$this->assertContains('邵栋', $names);
|
|
|
$titles = array_column(array_map(fn ($item) => $item->extra, $items), 'academic_title', null);
|
|
|
$this->assertSame('教授', $items[array_search('杨鲲', $names, true)]->extra['academic_title']);
|
|
|
$this->assertSame('副教授', $items[array_search('邵栋', $names, true)]->extra['academic_title']);
|
|
|
}
|
|
|
|
|
|
public function test_parse_research_direction_names_keeps_valid_utf8_with_chinese_separators(): void
|
|
|
{
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'parseResearchDirectionNames');
|
|
|
$method->setAccessible(true);
|
|
|
|
|
|
$names = $method->invoke(
|
|
|
$adapter,
|
|
|
'纳米材料与器件、先进功能涂层、轻质高强金属材料',
|
|
|
);
|
|
|
|
|
|
$this->assertSame(
|
|
|
['纳米材料与器件', '先进功能涂层', '轻质高强金属材料'],
|
|
|
$names,
|
|
|
);
|
|
|
foreach ($names as $name) {
|
|
|
$this->assertTrue(mb_check_encoding($name, 'UTF-8'));
|
|
|
}
|
|
|
$this->assertNotFalse(json_encode(['research_direction_names' => $names], JSON_THROW_ON_ERROR));
|
|
|
}
|
|
|
|
|
|
public function test_smse_long_research_li_items_are_kept_as_whole_and_json_safe(): void
|
|
|
{
|
|
|
$html = <<<'HTML'
|
|
|
<div class="panel-head"><div class="title">研究方向</div></div>
|
|
|
<div class="panel-body">
|
|
|
<ul>
|
|
|
<li>研究方向一:高性能陶瓷基复合材料及其构件制备</li>
|
|
|
<li>已形成完整的研究体系并在多个领域取得突破</li>
|
|
|
<li>金属材料强韧化与表面改性</li>
|
|
|
</ul>
|
|
|
</div>
|
|
|
HTML;
|
|
|
|
|
|
$adapter = new FacultyListHtmlAdapter;
|
|
|
$method = new \ReflectionMethod($adapter, 'applyProfileMetadataToItem');
|
|
|
$method->setAccessible(true);
|
|
|
$item = $method->invoke(
|
|
|
$adapter,
|
|
|
new \App\Services\Crawl\CrawlItemDto(
|
|
|
externalId: 'faculty:smse-utf8',
|
|
|
title: '赵斌元',
|
|
|
canonicalUrl: 'https://smse.sjtu.edu.cn/people/detail_new/1',
|
|
|
extra: ['lead_author' => ['name' => '赵斌元']],
|
|
|
),
|
|
|
$html,
|
|
|
);
|
|
|
|
|
|
$names = $item->extra['research_direction_names'];
|
|
|
$this->assertSame(
|
|
|
['高性能陶瓷基复合材料及其构件制备', '金属材料强韧化与表面改性'],
|
|
|
$names,
|
|
|
);
|
|
|
$this->assertNotFalse(json_encode(['research_direction_names' => $names], JSON_THROW_ON_ERROR));
|
|
|
}
|
|
|
}
|