|
|
<?php
|
|
|
|
|
|
namespace App\Services\Crawl;
|
|
|
|
|
|
use App\Models\CrawlJob;
|
|
|
use App\Models\CrawlJobItem;
|
|
|
use App\Models\DictItem;
|
|
|
use App\Models\DictType;
|
|
|
use App\Models\News;
|
|
|
use App\Models\Paper;
|
|
|
use App\Models\Teacher;
|
|
|
use App\Models\University;
|
|
|
use App\Services\ResearchDirectionResolver;
|
|
|
use App\Support\TeacherLibraryStatus;
|
|
|
use Carbon\Carbon;
|
|
|
use Illuminate\Support\Facades\DB;
|
|
|
use Illuminate\Support\Facades\Log;
|
|
|
|
|
|
class CrawlImportService
|
|
|
{
|
|
|
/**
|
|
|
* @param array<int>|null $itemIds
|
|
|
* @param array{university_id?:int,department?:string,city?:string,research_direction_ids?:int[]} $teacherDefaults
|
|
|
* @return array{
|
|
|
* imported:int,
|
|
|
* imported_primary:int,
|
|
|
* skipped:int,
|
|
|
* failed:int,
|
|
|
* papers_imported:int,
|
|
|
* teacher_leads_imported:int,
|
|
|
* teachers_imported:int,
|
|
|
* teachers_updated:int,
|
|
|
* news_imported:int
|
|
|
* }
|
|
|
*/
|
|
|
public function import(
|
|
|
CrawlJob $job,
|
|
|
?array $itemIds = null,
|
|
|
bool $selectAll = false,
|
|
|
array $teacherDefaults = [],
|
|
|
array $newsDefaults = [],
|
|
|
): array {
|
|
|
$newsDefaults = $this->enrichNewsDefaults($job, $newsDefaults);
|
|
|
$query = CrawlJobItem::query()
|
|
|
->where('crawl_job_id', $job->id)
|
|
|
->whereIn('status', ['preview']);
|
|
|
|
|
|
if (! $selectAll && $itemIds !== null && $itemIds !== []) {
|
|
|
$query->whereIn('id', $itemIds);
|
|
|
}
|
|
|
|
|
|
$items = $query->orderBy('id')->get();
|
|
|
$imported = 0;
|
|
|
$skipped = 0;
|
|
|
$failed = 0;
|
|
|
$papersImported = 0;
|
|
|
$teacherLeadsImported = 0;
|
|
|
$teachersImported = 0;
|
|
|
$teachersUpdated = 0;
|
|
|
$newsImported = 0;
|
|
|
|
|
|
DB::transaction(function () use (
|
|
|
$items,
|
|
|
$job,
|
|
|
$teacherDefaults,
|
|
|
$newsDefaults,
|
|
|
&$imported,
|
|
|
&$skipped,
|
|
|
&$failed,
|
|
|
&$papersImported,
|
|
|
&$teacherLeadsImported,
|
|
|
&$teachersImported,
|
|
|
&$teachersUpdated,
|
|
|
&$newsImported,
|
|
|
) {
|
|
|
foreach ($items as $item) {
|
|
|
try {
|
|
|
$result = match ($item->target_type) {
|
|
|
'paper' => ['id' => $this->importPaper($job, $item), 'updated' => false],
|
|
|
'teacher_lead', 'teacher' => $this->importTeacher($job, $item, $teacherDefaults),
|
|
|
default => ['id' => $this->importNews($job, $item, $newsDefaults), 'updated' => false],
|
|
|
};
|
|
|
$id = is_array($result) ? ($result['id'] ?? null) : $result;
|
|
|
$updated = is_array($result) ? (bool) ($result['updated'] ?? false) : false;
|
|
|
|
|
|
if ($id) {
|
|
|
$item->update(['status' => 'imported', 'target_id' => $id]);
|
|
|
$imported++;
|
|
|
match ($item->target_type) {
|
|
|
'paper' => $papersImported++,
|
|
|
'teacher_lead' => $teacherLeadsImported++,
|
|
|
'teacher' => $updated ? $teachersUpdated++ : $teachersImported++,
|
|
|
default => $newsImported++,
|
|
|
};
|
|
|
} else {
|
|
|
if ($item->status === 'preview') {
|
|
|
$item->update(['status' => 'skipped']);
|
|
|
}
|
|
|
$skipped++;
|
|
|
}
|
|
|
} catch (\Throwable $e) {
|
|
|
$failed++;
|
|
|
Log::warning('crawl_import_item_failed', [
|
|
|
'crawl_job_id' => $job->id,
|
|
|
'item_id' => $item->id,
|
|
|
'target_type' => $item->target_type,
|
|
|
'message' => $e->getMessage(),
|
|
|
]);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
$job->update([
|
|
|
'items_imported' => $this->countPrimaryImported($job),
|
|
|
]);
|
|
|
});
|
|
|
|
|
|
return [
|
|
|
'imported' => $imported,
|
|
|
'imported_primary' => $this->countPrimaryImported($job),
|
|
|
'skipped' => $skipped,
|
|
|
'failed' => $failed,
|
|
|
'papers_imported' => $papersImported,
|
|
|
'teacher_leads_imported' => $teacherLeadsImported,
|
|
|
'teachers_imported' => $teachersImported,
|
|
|
'teachers_updated' => $teachersUpdated,
|
|
|
'news_imported' => $newsImported,
|
|
|
];
|
|
|
}
|
|
|
|
|
|
protected function countPrimaryImported(CrawlJob $job): int
|
|
|
{
|
|
|
$targetType = match ($job->target_type) {
|
|
|
'paper' => 'paper',
|
|
|
'teacher' => 'teacher',
|
|
|
default => 'news',
|
|
|
};
|
|
|
|
|
|
return CrawlJobItem::query()
|
|
|
->where('crawl_job_id', $job->id)
|
|
|
->where('target_type', $targetType)
|
|
|
->where('status', 'imported')
|
|
|
->count();
|
|
|
}
|
|
|
|
|
|
protected function importPaper(CrawlJob $job, CrawlJobItem $item): ?int
|
|
|
{
|
|
|
$payload = $item->payload ?? [];
|
|
|
$externalId = $item->external_id;
|
|
|
$title = trim((string) $item->title);
|
|
|
if ($title === '') {
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
$existing = Paper::query()
|
|
|
->where('external_id', $externalId)
|
|
|
->where('source', 'crawl')
|
|
|
->first();
|
|
|
if ($existing) {
|
|
|
$item->update(['status' => 'duplicate']);
|
|
|
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
$paper = Paper::query()->create([
|
|
|
'title' => $title,
|
|
|
'authors' => $payload['authors'] ?? null,
|
|
|
'school_name' => $payload['school_name'] ?? null,
|
|
|
'published_at' => $this->normalizePaperPublishedAt($payload['published_at'] ?? null),
|
|
|
'url' => $item->canonical_url,
|
|
|
'summary' => $payload['summary'] ?? null,
|
|
|
'source' => 'crawl',
|
|
|
'external_id' => $externalId,
|
|
|
'source_site' => $item->source_name,
|
|
|
'crawl_job_id' => $job->id,
|
|
|
]);
|
|
|
|
|
|
return $paper->id;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* @param array{university_id?:int,department?:string,city?:string,research_direction_ids?:int[]} $defaults
|
|
|
* @return array{id:?int,updated:bool}
|
|
|
*/
|
|
|
protected function importTeacher(CrawlJob $job, CrawlJobItem $item, array $defaults = []): array
|
|
|
{
|
|
|
$payload = $item->payload ?? [];
|
|
|
$lead = $payload['lead_author'] ?? null;
|
|
|
if (! is_array($lead)) {
|
|
|
$lead = [
|
|
|
'name' => $item->title,
|
|
|
'email' => null,
|
|
|
'affiliation' => null,
|
|
|
'university_name' => $payload['school_name'] ?? null,
|
|
|
];
|
|
|
}
|
|
|
|
|
|
$name = trim((string) ($lead['name'] ?? ''));
|
|
|
if ($name === '') {
|
|
|
return ['id' => null, 'updated' => false];
|
|
|
}
|
|
|
|
|
|
$email = CrawlAuthorParser::normalizeEmail($lead['email'] ?? null);
|
|
|
$phone = trim((string) ($lead['phone'] ?? $payload['phone'] ?? ''));
|
|
|
$academicTitle = trim((string) ($lead['academic_title'] ?? $payload['academic_title'] ?? ''));
|
|
|
$collegeName = trim((string) ($lead['college'] ?? $lead['affiliation'] ?? $payload['college_name'] ?? ''));
|
|
|
$defaultDepartment = trim((string) ($defaults['department'] ?? ''));
|
|
|
if ($defaultDepartment !== '') {
|
|
|
$collegeName = $defaultDepartment;
|
|
|
}
|
|
|
$bio = trim((string) ($lead['bio'] ?? $payload['bio'] ?? ''));
|
|
|
$profileUrl = trim((string) ($payload['profile_url'] ?? $item->canonical_url ?? ''));
|
|
|
|
|
|
$universityId = isset($defaults['university_id']) ? (int) $defaults['university_id'] : null;
|
|
|
$city = isset($defaults['city']) ? trim((string) $defaults['city']) : null;
|
|
|
if ($universityId && ! $city) {
|
|
|
$city = University::query()->whereKey($universityId)->value('city');
|
|
|
}
|
|
|
$leadUniversityName = trim((string) ($lead['university_name'] ?? ''));
|
|
|
if (! $universityId && $leadUniversityName !== '') {
|
|
|
$universityId = $this->resolveUniversityId($leadUniversityName);
|
|
|
if ($universityId) {
|
|
|
$city = $city ?: University::query()->whereKey($universityId)->value('city');
|
|
|
}
|
|
|
}
|
|
|
|
|
|
$directionIds = $this->resolveTeacherDirectionIds($payload, $lead, $defaults);
|
|
|
|
|
|
$existing = $this->findExistingTeacher(
|
|
|
$name,
|
|
|
$email,
|
|
|
$profileUrl,
|
|
|
$universityId,
|
|
|
$leadUniversityName,
|
|
|
$collegeName,
|
|
|
);
|
|
|
|
|
|
if ($existing) {
|
|
|
$updated = $this->fillEmptyTeacherFields($existing, [
|
|
|
'email' => $email,
|
|
|
'phone' => $phone !== '' ? $phone : null,
|
|
|
'title' => $academicTitle !== '' ? $academicTitle : null,
|
|
|
'bio' => $bio !== '' ? $bio : null,
|
|
|
'university_id' => $universityId,
|
|
|
'department' => $collegeName !== '' ? $collegeName : null,
|
|
|
'city' => $city,
|
|
|
'profile_url' => $profileUrl !== '' ? $profileUrl : null,
|
|
|
'research_direction_ids' => $directionIds,
|
|
|
]);
|
|
|
|
|
|
if ($updated) {
|
|
|
return ['id' => $existing->id, 'updated' => true];
|
|
|
}
|
|
|
|
|
|
$item->update(['status' => 'duplicate']);
|
|
|
|
|
|
return ['id' => null, 'updated' => false];
|
|
|
}
|
|
|
|
|
|
$sourceId = $this->resolveTeacherSourceId($item->target_type);
|
|
|
if (! $sourceId) {
|
|
|
throw new \RuntimeException('老师库字典未配置');
|
|
|
}
|
|
|
|
|
|
$remarkParts = [
|
|
|
match ($item->target_type) {
|
|
|
'teacher_lead' => '论文库入库',
|
|
|
'teacher' => '高校抓取入库',
|
|
|
default => '爬虫入库',
|
|
|
},
|
|
|
];
|
|
|
if ($profileUrl !== '') {
|
|
|
$remarkParts[] = '主页：'.$profileUrl;
|
|
|
}
|
|
|
|
|
|
$universityText = null;
|
|
|
if (! $universityId) {
|
|
|
if ($leadUniversityName !== '' && $collegeName !== '') {
|
|
|
$universityText = $leadUniversityName.' · '.$collegeName;
|
|
|
} elseif ($leadUniversityName !== '') {
|
|
|
$universityText = $leadUniversityName;
|
|
|
} elseif ($collegeName !== '') {
|
|
|
$universityText = $collegeName;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
$libraryStatus = $item->target_type === 'teacher_lead'
|
|
|
? TeacherLibraryStatus::Pending
|
|
|
: TeacherLibraryStatus::Active;
|
|
|
|
|
|
$teacher = Teacher::query()->create([
|
|
|
'name' => $name,
|
|
|
'university_id' => $universityId,
|
|
|
'university_text' => $universityText,
|
|
|
'department' => $collegeName !== '' ? $collegeName : null,
|
|
|
'bio' => $bio !== '' ? $bio : null,
|
|
|
'city' => $city ?: '待补充',
|
|
|
'title' => $academicTitle !== '' ? $academicTitle : '待补充',
|
|
|
'email' => $email,
|
|
|
'phone' => $phone !== '' ? $phone : null,
|
|
|
'source_dict_item_id' => $sourceId,
|
|
|
'status_dict_item_id' => null,
|
|
|
'library_status' => $libraryStatus,
|
|
|
'remark' => implode('；', $remarkParts),
|
|
|
]);
|
|
|
|
|
|
if ($directionIds !== []) {
|
|
|
$teacher->researchDirections()->sync($directionIds);
|
|
|
}
|
|
|
|
|
|
$paperExternalId = $payload['paper_external_id'] ?? null;
|
|
|
if ($paperExternalId) {
|
|
|
$paper = Paper::query()
|
|
|
->where('external_id', $paperExternalId)
|
|
|
->where('source', 'crawl')
|
|
|
->first();
|
|
|
if ($paper) {
|
|
|
$teacher->papers()->syncWithoutDetaching([$paper->id]);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
return ['id' => $teacher->id, 'updated' => false];
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* @param array<string, mixed> $payload
|
|
|
* @param array<string, mixed> $lead
|
|
|
* @param array{university_id?:int,department?:string,city?:string,research_direction_ids?:int[]} $defaults
|
|
|
* @return list<int>
|
|
|
*/
|
|
|
protected function resolveTeacherDirectionIds(array $payload, array $lead, array $defaults): array
|
|
|
{
|
|
|
if (! empty($defaults['research_direction_ids']) && is_array($defaults['research_direction_ids'])) {
|
|
|
return array_values(array_unique(array_map('intval', $defaults['research_direction_ids'])));
|
|
|
}
|
|
|
|
|
|
$directionNames = $payload['research_direction_names'] ?? $lead['research_direction_names'] ?? [];
|
|
|
if (is_string($directionNames)) {
|
|
|
$directionNames = preg_split('/(?:、|，|,|；|;|\/)+/u', $directionNames) ?: [];
|
|
|
}
|
|
|
if (! is_array($directionNames) || $directionNames === []) {
|
|
|
return [];
|
|
|
}
|
|
|
|
|
|
return app(ResearchDirectionResolver::class)->resolveIds([], $directionNames);
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 仅补空字段：已有值不覆盖。
|
|
|
*
|
|
|
* @param array{
|
|
|
* email?:?string,
|
|
|
* phone?:?string,
|
|
|
* title?:?string,
|
|
|
* bio?:?string,
|
|
|
* university_id?:?int,
|
|
|
* department?:?string,
|
|
|
* city?:?string,
|
|
|
* profile_url?:?string,
|
|
|
* research_direction_ids?:list<int>
|
|
|
* } $incoming
|
|
|
*/
|
|
|
public function fillEmptyTeacherFields(Teacher $teacher, array $incoming): bool
|
|
|
{
|
|
|
$updates = [];
|
|
|
$changed = false;
|
|
|
|
|
|
$email = CrawlAuthorParser::normalizeEmail($incoming['email'] ?? null);
|
|
|
if ($this->isBlankTeacherValue($teacher->email) && $email) {
|
|
|
$emailTaken = Teacher::query()
|
|
|
->where('email', $email)
|
|
|
->where('id', '!=', $teacher->id)
|
|
|
->exists();
|
|
|
if (! $emailTaken) {
|
|
|
$updates['email'] = $email;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
$phone = trim((string) ($incoming['phone'] ?? ''));
|
|
|
if ($this->isBlankTeacherValue($teacher->phone) && $phone !== '') {
|
|
|
$updates['phone'] = $phone;
|
|
|
}
|
|
|
|
|
|
$title = trim((string) ($incoming['title'] ?? ''));
|
|
|
if ($title !== '' && CrawlAuthorParser::looksLikeAcademicTitle($title)
|
|
|
&& ($this->isBlankTeacherValue($teacher->title) || ! CrawlAuthorParser::looksLikeAcademicTitle((string) $teacher->title))) {
|
|
|
$updates['title'] = $title;
|
|
|
}
|
|
|
|
|
|
$bio = trim((string) ($incoming['bio'] ?? ''));
|
|
|
if ($this->isBlankTeacherValue($teacher->bio) && $bio !== '') {
|
|
|
$updates['bio'] = $bio;
|
|
|
}
|
|
|
|
|
|
$department = trim((string) ($incoming['department'] ?? ''));
|
|
|
if ($this->isBlankTeacherValue($teacher->department) && $department !== '') {
|
|
|
$updates['department'] = $department;
|
|
|
}
|
|
|
|
|
|
$city = trim((string) ($incoming['city'] ?? ''));
|
|
|
if ($this->isBlankTeacherValue($teacher->city) && $city !== '') {
|
|
|
$updates['city'] = $city;
|
|
|
}
|
|
|
|
|
|
$universityId = isset($incoming['university_id']) ? (int) $incoming['university_id'] : 0;
|
|
|
if (! $teacher->university_id && $universityId > 0) {
|
|
|
$updates['university_id'] = $universityId;
|
|
|
}
|
|
|
|
|
|
$profileUrl = trim((string) ($incoming['profile_url'] ?? ''));
|
|
|
if ($profileUrl !== '') {
|
|
|
$remark = (string) ($teacher->remark ?? '');
|
|
|
if ($remark === '' || ! str_contains($remark, $profileUrl)) {
|
|
|
$updates['remark'] = trim($remark.($remark !== '' ? '；' : '').'主页：'.$profileUrl, '；');
|
|
|
}
|
|
|
}
|
|
|
|
|
|
if ($updates !== []) {
|
|
|
$teacher->fill($updates);
|
|
|
$teacher->save();
|
|
|
$changed = true;
|
|
|
}
|
|
|
|
|
|
$directionIds = array_values(array_unique(array_filter(array_map(
|
|
|
'intval',
|
|
|
$incoming['research_direction_ids'] ?? [],
|
|
|
))));
|
|
|
if ($directionIds !== [] && $teacher->researchDirections()->count() === 0) {
|
|
|
$teacher->researchDirections()->sync($directionIds);
|
|
|
$changed = true;
|
|
|
}
|
|
|
|
|
|
return $changed;
|
|
|
}
|
|
|
|
|
|
protected function isBlankTeacherValue(mixed $value): bool
|
|
|
{
|
|
|
$text = trim((string) ($value ?? ''));
|
|
|
|
|
|
return $text === '' || $text === '待补充';
|
|
|
}
|
|
|
|
|
|
protected function findExistingTeacher(
|
|
|
string $name,
|
|
|
?string $email,
|
|
|
?string $profileUrl,
|
|
|
?int $universityId,
|
|
|
string $leadUniversityName,
|
|
|
?string $collegeName,
|
|
|
): ?Teacher {
|
|
|
if ($email) {
|
|
|
$byEmail = Teacher::query()->where('email', $email)->first();
|
|
|
if ($byEmail) {
|
|
|
return $byEmail;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
if ($profileUrl !== null && $profileUrl !== '') {
|
|
|
$byProfile = Teacher::query()->where('remark', 'like', '%'.$profileUrl.'%')->first();
|
|
|
if ($byProfile) {
|
|
|
return $byProfile;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
$dupQuery = Teacher::query()->where('name', $name);
|
|
|
if ($collegeName !== null && $collegeName !== '') {
|
|
|
$dupQuery->where('department', $collegeName);
|
|
|
}
|
|
|
if ($universityId) {
|
|
|
$dupQuery->where('university_id', $universityId);
|
|
|
} elseif ($leadUniversityName !== '') {
|
|
|
$dupQuery->where(function ($q) use ($leadUniversityName) {
|
|
|
$q->where('university_text', $leadUniversityName)
|
|
|
->orWhere('university_text', 'like', $leadUniversityName.'%');
|
|
|
});
|
|
|
}
|
|
|
|
|
|
return $dupQuery->first();
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* @param array{source?:string, category_dict_item_id?:int} $newsDefaults
|
|
|
*/
|
|
|
protected function importNews(CrawlJob $job, CrawlJobItem $item, array $newsDefaults = []): ?int
|
|
|
{
|
|
|
$url = $item->canonical_url;
|
|
|
if ($url && News::query()->where('source_url', $url)->exists()) {
|
|
|
$item->update(['status' => 'duplicate']);
|
|
|
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
$title = trim((string) $item->title);
|
|
|
if ($title === '') {
|
|
|
throw new \RuntimeException('标题为空，无法入库');
|
|
|
}
|
|
|
|
|
|
$payload = $item->payload ?? [];
|
|
|
$extra = $payload['extra'] ?? [];
|
|
|
$categoryId = 0;
|
|
|
if (! empty($newsDefaults['category_dict_item_id'])) {
|
|
|
$categoryId = (int) $newsDefaults['category_dict_item_id'];
|
|
|
} elseif (isset($extra['category_dict_item_id'])) {
|
|
|
$categoryId = (int) $extra['category_dict_item_id'];
|
|
|
}
|
|
|
if ($categoryId <= 0) {
|
|
|
$categoryId = (int) (app(NewsCategoryMatcher::class)->resolveCategoryId(
|
|
|
$item->title,
|
|
|
$payload['summary'] ?? null,
|
|
|
$extra['keywords'] ?? CrawlKeywordParser::parse((string) ($job->keyword ?? '')),
|
|
|
) ?? 0);
|
|
|
}
|
|
|
if ($categoryId <= 0) {
|
|
|
$categoryId = null;
|
|
|
}
|
|
|
|
|
|
$content = app(NewsHtmlImageLocalizer::class)->localize(
|
|
|
$payload['content_html'] ?? '',
|
|
|
$url
|
|
|
) ?? '';
|
|
|
if ($content === '') {
|
|
|
$content = null;
|
|
|
}
|
|
|
|
|
|
$importSource = trim((string) ($extra['import_source'] ?? $newsDefaults['source'] ?? ''));
|
|
|
if ($importSource === '') {
|
|
|
$importSource = app(CrawlAddressSourceResolver::class)->resolveForNews(
|
|
|
$job->request_url ?? $job->platform_url,
|
|
|
$url,
|
|
|
) ?? '';
|
|
|
}
|
|
|
if ($importSource === '') {
|
|
|
$fallback = trim((string) ($item->source_name ?? ''));
|
|
|
$importSource = in_array($fallback, app(CrawlAddressSourceResolver::class)->genericAdapterSourceNames(), true)
|
|
|
? '爬虫采集'
|
|
|
: ($fallback ?: '爬虫采集');
|
|
|
}
|
|
|
|
|
|
$news = News::query()->create([
|
|
|
'title' => $title,
|
|
|
'category_dict_item_id' => $categoryId,
|
|
|
'source' => $importSource,
|
|
|
'source_url' => $url,
|
|
|
'source_site' => $importSource,
|
|
|
'crawl_job_id' => $job->id,
|
|
|
'summary' => $payload['summary'] ?? null,
|
|
|
'content_html' => $content,
|
|
|
'status' => 1,
|
|
|
'published_at' => $this->resolvePublishedAt($payload['published_at'] ?? null),
|
|
|
]);
|
|
|
|
|
|
return $news->id;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* @param array{source?:string, category_dict_item_id?:int} $newsDefaults
|
|
|
* @return array{source?:string, category_dict_item_id?:int}
|
|
|
*/
|
|
|
protected function enrichNewsDefaults(CrawlJob $job, array $newsDefaults): array
|
|
|
{
|
|
|
if (trim((string) ($newsDefaults['source'] ?? '')) !== '') {
|
|
|
return $newsDefaults;
|
|
|
}
|
|
|
|
|
|
$addressSource = app(CrawlAddressSourceResolver::class)->resolveByRequestUrl(
|
|
|
$job->request_url ?? $job->platform_url,
|
|
|
'industry_news',
|
|
|
);
|
|
|
if ($addressSource) {
|
|
|
$newsDefaults['source'] = $addressSource;
|
|
|
}
|
|
|
|
|
|
return $newsDefaults;
|
|
|
}
|
|
|
|
|
|
protected function resolvePublishedAt(mixed $value): Carbon
|
|
|
{
|
|
|
if ($value === null || $value === '') {
|
|
|
return now();
|
|
|
}
|
|
|
|
|
|
try {
|
|
|
return Carbon::parse($value);
|
|
|
} catch (\Throwable) {
|
|
|
$normalized = HtmlCrawlSupport::normalizeDate((string) $value);
|
|
|
|
|
|
return $normalized ? Carbon::parse($normalized.' 00:00:00') : now();
|
|
|
}
|
|
|
}
|
|
|
|
|
|
protected function normalizePaperPublishedAt(mixed $value): ?string
|
|
|
{
|
|
|
if ($value === null || $value === '') {
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
try {
|
|
|
return Carbon::parse($value)->toDateString();
|
|
|
} catch (\Throwable) {
|
|
|
return HtmlCrawlSupport::normalizeDate((string) $value);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
protected function resolveUniversityId(string $name): ?int
|
|
|
{
|
|
|
$name = trim($name);
|
|
|
if ($name === '') {
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
$id = University::query()
|
|
|
->where('name', $name)
|
|
|
->where('status', 1)
|
|
|
->value('id');
|
|
|
|
|
|
return $id ? (int) $id : null;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 论文任务第一作者 → 论文库；师资列表任务 → 高校抓取；其余 → 手动录入。
|
|
|
*/
|
|
|
protected function resolveTeacherSourceId(string $targetType): ?int
|
|
|
{
|
|
|
$value = match ($targetType) {
|
|
|
'teacher_lead' => 'paper',
|
|
|
'teacher' => 'faculty_crawl',
|
|
|
default => 'manual',
|
|
|
};
|
|
|
|
|
|
$typeId = DictType::query()->where('code', 'teacher_source')->where('status', 1)->value('id');
|
|
|
if (! $typeId) {
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
return DictItem::query()
|
|
|
->where('dict_type_id', $typeId)
|
|
|
->where('value', $value)
|
|
|
->where('status', 1)
|
|
|
->value('id');
|
|
|
}
|
|
|
|
|
|
protected function defaultTeacherStatusId(): ?int
|
|
|
{
|
|
|
$typeId = DictType::query()->where('code', 'teacher_status')->where('status', 1)->value('id');
|
|
|
if (! $typeId) {
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
return DictItem::query()
|
|
|
->where('dict_type_id', $typeId)
|
|
|
->where('value', 'active')
|
|
|
->where('status', 1)
|
|
|
->value('id');
|
|
|
}
|
|
|
}
|
