slake-school-service/app/Services/Crawl/CrawlImportService.php

<?php

namespace App\Services\Crawl;

use App\Models\CrawlJob;
use App\Models\CrawlJobItem;
use App\Models\DictItem;
use App\Models\DictType;
use App\Models\News;
use App\Models\Paper;
use App\Models\Teacher;
use App\Models\University;
use Carbon\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;

class CrawlImportService
{
    /**
     * @param  array<int>|null  $itemIds
     * @param  array{university_id?:int,city?:string,research_direction_ids?:int[]}  $teacherDefaults
     * @return array{imported:int, skipped:int, failed:int}
     */
    public function import(
        CrawlJob $job,
        ?array $itemIds = null,
        bool $selectAll = false,
        array $teacherDefaults = [],
        array $newsDefaults = [],
    ): array {
        $query = CrawlJobItem::query()
            ->where('crawl_job_id', $job->id)
            ->whereIn('status', ['preview']);

        if (! $selectAll && $itemIds !== null && $itemIds !== []) {
            $query->whereIn('id', $itemIds);
        }

        $items = $query->get();
        $imported = 0;
        $skipped = 0;
        $failed = 0;

        DB::transaction(function () use ($items, $job, $teacherDefaults, $newsDefaults, &$imported, &$skipped, &$failed) {
            foreach ($items as $item) {
                try {
                    $id = match ($item->target_type) {
                        'paper' => $this->importPaper($job, $item),
                        'teacher_lead', 'teacher' => $this->importTeacher($job, $item, $teacherDefaults),
                        default => $this->importNews($job, $item, $newsDefaults),
                    };
                    if ($id) {
                        $item->update(['status' => 'imported', 'target_id' => $id]);
                        $imported++;
                    } else {
                        if ($item->status === 'preview') {
                            $item->update(['status' => 'skipped']);
                        }
                        $skipped++;
                    }
                } catch (\Throwable $e) {
                    $failed++;
                    Log::warning('crawl_import_item_failed', [
                        'crawl_job_id' => $job->id,
                        'item_id' => $item->id,
                        'target_type' => $item->target_type,
                        'message' => $e->getMessage(),
                    ]);
                }
            }

            $job->update([
                'items_imported' => CrawlJobItem::query()
                    ->where('crawl_job_id', $job->id)
                    ->where('status', 'imported')
                    ->count(),
            ]);
        });

        return compact('imported', 'skipped', 'failed');
    }

    protected function importPaper(CrawlJob $job, CrawlJobItem $item): ?int
    {
        $payload = $item->payload ?? [];
        $externalId = $item->external_id;

        $existing = Paper::query()
            ->where('external_id', $externalId)
            ->where('source', 'crawl')
            ->first();
        if ($existing) {
            $item->update(['status' => 'duplicate']);

            return null;
        }

        $paper = Paper::query()->create([
            'title' => $item->title,
            'authors' => $payload['authors'] ?? null,
            'school_name' => $payload['school_name'] ?? null,
            'published_at' => $payload['published_at'] ?? null,
            'url' => $item->canonical_url,
            'summary' => $payload['summary'] ?? null,
            'source' => 'crawl',
            'external_id' => $externalId,
            'source_site' => $item->source_name,
            'crawl_job_id' => $job->id,
        ]);

        return $paper->id;
    }

    /**
     * @param  array{university_id?:int,city?:string,research_direction_ids?:int[]}  $defaults
     */
    protected function importTeacher(CrawlJob $job, CrawlJobItem $item, array $defaults = []): ?int
    {
        $payload = $item->payload ?? [];
        $lead = $payload['lead_author'] ?? null;
        if (! is_array($lead)) {
            $lead = [
                'name' => $item->title,
                'email' => null,
                'affiliation' => null,
                'university_name' => $payload['school_name'] ?? null,
            ];
        }

        $name = trim((string) ($lead['name'] ?? ''));
        if ($name === '') {
            return null;
        }

        $email = CrawlAuthorParser::normalizeEmail($lead['email'] ?? null);
        if ($email && Teacher::query()->where('email', $email)->exists()) {
            $item->update(['status' => 'duplicate']);

            return null;
        }

        $academicTitle = trim((string) ($lead['academic_title'] ?? $payload['academic_title'] ?? ''));
        $collegeName = trim((string) ($lead['college'] ?? $lead['affiliation'] ?? $payload['college_name'] ?? ''));
        $profileUrl = trim((string) ($payload['profile_url'] ?? $item->canonical_url ?? ''));

        $universityId = isset($defaults['university_id']) ? (int) $defaults['university_id'] : null;
        $city = isset($defaults['city']) ? trim((string) $defaults['city']) : null;
        if ($universityId && ! $city) {
            $city = University::query()->whereKey($universityId)->value('city');
        }
        $leadUniversityName = trim((string) ($lead['university_name'] ?? ''));
        if (! $universityId && $leadUniversityName !== '') {
            $universityId = $this->resolveUniversityId($leadUniversityName);
            if ($universityId) {
                $city = $city ?: University::query()->whereKey($universityId)->value('city');
            }
        }

        if ($this->teacherAlreadyExists($name, $email, $profileUrl, $universityId, $leadUniversityName, $collegeName)) {
            $item->update(['status' => 'duplicate']);

            return null;
        }

        $sourceId = $this->resolveTeacherSourceId($item->target_type);
        $statusId = $this->defaultTeacherStatusId();
        if (! $sourceId || ! $statusId) {
            throw new \RuntimeException('老师库字典未配置');
        }

        $remarkParts = [
            match ($item->target_type) {
                'teacher_lead' => '论文库入库',
                'teacher' => '高校抓取入库',
                default => '爬虫入库',
            },
        ];
        if ($profileUrl !== '') {
            $remarkParts[] = '主页：'.$profileUrl;
        }

        $universityText = null;
        if (! $universityId) {
            if ($leadUniversityName !== '' && $collegeName !== '') {
                $universityText = $leadUniversityName.' · '.$collegeName;
            } elseif ($leadUniversityName !== '') {
                $universityText = $leadUniversityName;
            } elseif ($collegeName !== '') {
                $universityText = $collegeName;
            }
        }

        $teacher = Teacher::query()->create([
            'name' => $name,
            'university_id' => $universityId,
            'university_text' => $universityText,
            'department' => $collegeName !== '' ? $collegeName : null,
            'city' => $city ?: '待补充',
            'title' => $academicTitle !== '' ? $academicTitle : '待补充',
            'email' => $email,
            'source_dict_item_id' => $sourceId,
            'status_dict_item_id' => $statusId,
            'remark' => implode('；', $remarkParts),
        ]);

        if (! empty($defaults['research_direction_ids']) && is_array($defaults['research_direction_ids'])) {
            $teacher->researchDirections()->sync($defaults['research_direction_ids']);
        }

        $paperExternalId = $payload['paper_external_id'] ?? null;
        if ($paperExternalId) {
            $paper = Paper::query()
                ->where('external_id', $paperExternalId)
                ->where('source', 'crawl')
                ->first();
            if ($paper) {
                $teacher->papers()->syncWithoutDetaching([$paper->id]);
            }
        }

        return $teacher->id;
    }

    /**
     * @param  array{source?:string, category_dict_item_id?:int}  $newsDefaults
     */
    protected function importNews(CrawlJob $job, CrawlJobItem $item, array $newsDefaults = []): ?int
    {
        $url = $item->canonical_url;
        if ($url && News::query()->where('source_url', $url)->exists()) {
            $item->update(['status' => 'duplicate']);

            return null;
        }

        $title = trim((string) $item->title);
        if ($title === '') {
            throw new \RuntimeException('标题为空，无法入库');
        }

        $payload = $item->payload ?? [];
        $extra = $payload['extra'] ?? [];
        $categoryId = isset($extra['category_dict_item_id']) ? (int) $extra['category_dict_item_id'] : 0;
        if ($categoryId <= 0 && ! empty($newsDefaults['category_dict_item_id'])) {
            $categoryId = (int) $newsDefaults['category_dict_item_id'];
        }
        if ($categoryId <= 0) {
            $categoryId = (int) (app(NewsCategoryMatcher::class)->resolveCategoryId(
                $item->title,
                $payload['summary'] ?? null,
                $extra['keywords'] ?? CrawlKeywordParser::parse((string) ($job->keyword ?? '')),
            ) ?? 0);
        }
        if ($categoryId <= 0) {
            $categoryId = null;
        }

        $content = app(NewsHtmlImageLocalizer::class)->localize(
            $payload['content_html'] ?? '',
            $url
        ) ?? '';
        if ($content === '') {
            $content = '<p>（爬虫抓取，请编辑正文）</p>';
        }

        $importSource = trim((string) ($extra['import_source'] ?? $newsDefaults['source'] ?? ''));
        if ($importSource === '') {
            $importSource = $item->source_name ?: '爬虫采集';
        }

        $news = News::query()->create([
            'title' => $title,
            'category_dict_item_id' => $categoryId,
            'source' => $importSource,
            'source_url' => $url,
            'source_site' => $importSource,
            'crawl_job_id' => $job->id,
            'summary' => $payload['summary'] ?? null,
            'content_html' => $content,
            'status' => 0,
            'published_at' => $this->resolvePublishedAt($payload['published_at'] ?? null),
        ]);

        return $news->id;
    }

    protected function resolvePublishedAt(mixed $value): Carbon
    {
        if ($value === null || $value === '') {
            return now();
        }

        try {
            return Carbon::parse($value);
        } catch (\Throwable) {
            $normalized = HtmlCrawlSupport::normalizeDate((string) $value);

            return $normalized ? Carbon::parse($normalized.' 00:00:00') : now();
        }
    }

    protected function teacherAlreadyExists(
        string $name,
        ?string $email,
        ?string $profileUrl,
        ?int $universityId,
        string $leadUniversityName,
        ?string $collegeName,
    ): bool {
        if ($email && Teacher::query()->where('email', $email)->exists()) {
            return true;
        }

        if ($email) {
            return false;
        }

        if ($profileUrl !== '') {
            return Teacher::query()->where('remark', 'like', '%'.$profileUrl.'%')->exists();
        }

        $dupQuery = Teacher::query()->where('name', $name);
        if ($collegeName !== '') {
            $dupQuery->where('department', $collegeName);
        }
        if ($universityId) {
            $dupQuery->where('university_id', $universityId);
        } elseif ($leadUniversityName !== '') {
            $dupQuery->where(function ($q) use ($leadUniversityName) {
                $q->where('university_text', $leadUniversityName)
                    ->orWhere('university_text', 'like', $leadUniversityName.'%');
            });
        }

        return $dupQuery->exists();
    }

    protected function resolveUniversityId(string $name): ?int
    {
        $name = trim($name);
        if ($name === '') {
            return null;
        }

        $id = University::query()
            ->where('name', $name)
            ->where('status', 1)
            ->value('id');

        return $id ? (int) $id : null;
    }

    /**
     * 论文任务第一作者 → 论文库；师资列表任务 → 高校抓取；其余 → 手动录入。
     */
    protected function resolveTeacherSourceId(string $targetType): ?int
    {
        $value = match ($targetType) {
            'teacher_lead' => 'paper',
            'teacher' => 'faculty_crawl',
            default => 'manual',
        };

        $typeId = DictType::query()->where('code', 'teacher_source')->where('status', 1)->value('id');
        if (! $typeId) {
            return null;
        }

        return DictItem::query()
            ->where('dict_type_id', $typeId)
            ->where('value', $value)
            ->where('status', 1)
            ->value('id');
    }

    protected function defaultTeacherStatusId(): ?int
    {
        $typeId = DictType::query()->where('code', 'teacher_status')->where('status', 1)->value('id');
        if (! $typeId) {
            return null;
        }

        return DictItem::query()
            ->where('dict_type_id', $typeId)
            ->where('value', 'active')
            ->where('status', 1)
            ->value('id');
    }
}