Add university scraper system with backend, frontend, and configs
- Add src/university_scraper module with scraper, analyzer, and CLI - Add backend FastAPI service with API endpoints and database models - Add frontend React app with university management pages - Add configs for Harvard, Manchester, and UCL universities - Add artifacts with various scraper implementations - Add Docker compose configuration for deployment - Update .gitignore to exclude generated files 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
yangxiaoyu-crypto
26
frontend/Dockerfile
Normal file
26
frontend/Dockerfile
Normal file
@ -0,0 +1,26 @@
|
||||
FROM node:20-alpine as builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 复制package文件
|
||||
COPY package*.json ./
|
||||
RUN npm install
|
||||
|
||||
# 复制源代码
|
||||
COPY . .
|
||||
|
||||
# 构建
|
||||
RUN npm run build
|
||||
|
||||
# 生产镜像
|
||||
FROM nginx:alpine
|
||||
|
||||
# 复制构建产物
|
||||
COPY --from=builder /app/dist /usr/share/nginx/html
|
||||
|
||||
# 复制nginx配置
|
||||
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
CMD ["nginx", "-g", "daemon off;"]
|
||||
12
frontend/index.html
Normal file
12
frontend/index.html
Normal file
@ -0,0 +1,12 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>大学爬虫系统</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
21
frontend/nginx.conf
Normal file
21
frontend/nginx.conf
Normal file
@ -0,0 +1,21 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name localhost;
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# 处理SPA路由
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
|
||||
# 代理API请求到后端
|
||||
location /api {
|
||||
proxy_pass http://backend:8000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection 'upgrade';
|
||||
proxy_set_header Host $host;
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
}
|
||||
}
|
||||
3051
frontend/package-lock.json
generated
Normal file
3051
frontend/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
26
frontend/package.json
Normal file
26
frontend/package.json
Normal file
@ -0,0 +1,26 @@
|
||||
{
|
||||
"name": "university-scraper-web",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc && vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"react-router-dom": "^6.20.0",
|
||||
"@tanstack/react-query": "^5.8.0",
|
||||
"axios": "^1.6.0",
|
||||
"antd": "^5.11.0",
|
||||
"@ant-design/icons": "^5.2.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^18.2.0",
|
||||
"@types/react-dom": "^18.2.0",
|
||||
"@vitejs/plugin-react": "^4.2.0",
|
||||
"typescript": "^5.3.0",
|
||||
"vite": "^5.0.0"
|
||||
}
|
||||
}
|
||||
75
frontend/src/App.tsx
Normal file
75
frontend/src/App.tsx
Normal file
@ -0,0 +1,75 @@
|
||||
/**
|
||||
* 主应用组件
|
||||
*/
|
||||
import { useState } from 'react'
|
||||
import { BrowserRouter, Routes, Route, Link, useNavigate } from 'react-router-dom'
|
||||
import { Layout, Menu, Typography } from 'antd'
|
||||
import { HomeOutlined, PlusOutlined, DatabaseOutlined } from '@ant-design/icons'
|
||||
import HomePage from './pages/HomePage'
|
||||
import AddUniversityPage from './pages/AddUniversityPage'
|
||||
import UniversityDetailPage from './pages/UniversityDetailPage'
|
||||
|
||||
const { Header, Content, Footer } = Layout
|
||||
const { Title } = Typography
|
||||
|
||||
function AppContent() {
|
||||
const navigate = useNavigate()
|
||||
const [current, setCurrent] = useState('home')
|
||||
|
||||
const menuItems = [
|
||||
{
|
||||
key: 'home',
|
||||
icon: <HomeOutlined />,
|
||||
label: '大学列表',
|
||||
onClick: () => navigate('/')
|
||||
},
|
||||
{
|
||||
key: 'add',
|
||||
icon: <PlusOutlined />,
|
||||
label: '添加大学',
|
||||
onClick: () => navigate('/add')
|
||||
}
|
||||
]
|
||||
|
||||
return (
|
||||
<Layout style={{ minHeight: '100vh' }}>
|
||||
<Header style={{ display: 'flex', alignItems: 'center', background: '#001529' }}>
|
||||
<div style={{ color: 'white', fontSize: '20px', fontWeight: 'bold', marginRight: '40px' }}>
|
||||
<DatabaseOutlined /> 大学爬虫系统
|
||||
</div>
|
||||
<Menu
|
||||
theme="dark"
|
||||
mode="horizontal"
|
||||
selectedKeys={[current]}
|
||||
items={menuItems}
|
||||
onClick={(e) => setCurrent(e.key)}
|
||||
style={{ flex: 1 }}
|
||||
/>
|
||||
</Header>
|
||||
|
||||
<Content style={{ padding: '24px', background: '#f5f5f5' }}>
|
||||
<div style={{ maxWidth: 1200, margin: '0 auto' }}>
|
||||
<Routes>
|
||||
<Route path="/" element={<HomePage />} />
|
||||
<Route path="/add" element={<AddUniversityPage />} />
|
||||
<Route path="/university/:id" element={<UniversityDetailPage />} />
|
||||
</Routes>
|
||||
</div>
|
||||
</Content>
|
||||
|
||||
<Footer style={{ textAlign: 'center', background: '#f5f5f5' }}>
|
||||
大学爬虫系统 ©2024 - 一键生成 & 一键爬取
|
||||
</Footer>
|
||||
</Layout>
|
||||
)
|
||||
}
|
||||
|
||||
function App() {
|
||||
return (
|
||||
<BrowserRouter>
|
||||
<AppContent />
|
||||
</BrowserRouter>
|
||||
)
|
||||
}
|
||||
|
||||
export default App
|
||||
29
frontend/src/index.css
Normal file
29
frontend/src/index.css
Normal file
@ -0,0 +1,29 @@
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
|
||||
.container {
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
}
|
||||
|
||||
.card-hover:hover {
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
|
||||
transition: box-shadow 0.3s;
|
||||
}
|
||||
|
||||
.status-pending { color: #faad14; }
|
||||
.status-analyzing { color: #1890ff; }
|
||||
.status-ready { color: #52c41a; }
|
||||
.status-running { color: #1890ff; }
|
||||
.status-completed { color: #52c41a; }
|
||||
.status-failed { color: #ff4d4f; }
|
||||
.status-error { color: #ff4d4f; }
|
||||
26
frontend/src/main.tsx
Normal file
26
frontend/src/main.tsx
Normal file
@ -0,0 +1,26 @@
|
||||
import React from 'react'
|
||||
import ReactDOM from 'react-dom/client'
|
||||
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
|
||||
import { ConfigProvider } from 'antd'
|
||||
import zhCN from 'antd/locale/zh_CN'
|
||||
import App from './App'
|
||||
import './index.css'
|
||||
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: {
|
||||
refetchOnWindowFocus: false,
|
||||
retry: 1
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
ReactDOM.createRoot(document.getElementById('root')!).render(
|
||||
<React.StrictMode>
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<ConfigProvider locale={zhCN}>
|
||||
<App />
|
||||
</ConfigProvider>
|
||||
</QueryClientProvider>
|
||||
</React.StrictMode>
|
||||
)
|
||||
165
frontend/src/pages/AddUniversityPage.tsx
Normal file
165
frontend/src/pages/AddUniversityPage.tsx
Normal file
@ -0,0 +1,165 @@
|
||||
/**
|
||||
* 添加大学页面 - 一键生成爬虫脚本
|
||||
*/
|
||||
import { useState } from 'react'
|
||||
import { useNavigate } from 'react-router-dom'
|
||||
import { useMutation } from '@tanstack/react-query'
|
||||
import {
|
||||
Card, Form, Input, Button, Typography, Steps, Result, Spin, message
|
||||
} from 'antd'
|
||||
import { GlobalOutlined, RocketOutlined, CheckCircleOutlined, LoadingOutlined } from '@ant-design/icons'
|
||||
import { scriptApi } from '../services/api'
|
||||
|
||||
const { Title, Text, Paragraph } = Typography
|
||||
|
||||
export default function AddUniversityPage() {
|
||||
const navigate = useNavigate()
|
||||
const [form] = Form.useForm()
|
||||
const [currentStep, setCurrentStep] = useState(0)
|
||||
const [universityId, setUniversityId] = useState<number | null>(null)
|
||||
|
||||
// 生成脚本
|
||||
const generateMutation = useMutation({
|
||||
mutationFn: scriptApi.generate,
|
||||
onSuccess: (response) => {
|
||||
const data = response.data
|
||||
setUniversityId(data.university_id)
|
||||
setCurrentStep(2)
|
||||
message.success('脚本生成成功!')
|
||||
},
|
||||
onError: (error: any) => {
|
||||
message.error(error.response?.data?.detail || '生成失败')
|
||||
setCurrentStep(0)
|
||||
}
|
||||
})
|
||||
|
||||
const handleSubmit = (values: { url: string; name?: string }) => {
|
||||
setCurrentStep(1)
|
||||
generateMutation.mutate({
|
||||
university_url: values.url,
|
||||
university_name: values.name
|
||||
})
|
||||
}
|
||||
|
||||
const stepItems = [
|
||||
{
|
||||
title: '输入信息',
|
||||
icon: <GlobalOutlined />
|
||||
},
|
||||
{
|
||||
title: '分析生成',
|
||||
icon: currentStep === 1 ? <LoadingOutlined /> : <RocketOutlined />
|
||||
},
|
||||
{
|
||||
title: '完成',
|
||||
icon: <CheckCircleOutlined />
|
||||
}
|
||||
]
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<Title level={3} style={{ textAlign: 'center', marginBottom: 32 }}>
|
||||
添加大学 - 一键生成爬虫脚本
|
||||
</Title>
|
||||
|
||||
<Steps current={currentStep} items={stepItems} style={{ marginBottom: 40 }} />
|
||||
|
||||
{currentStep === 0 && (
|
||||
<div style={{ maxWidth: 500, margin: '0 auto' }}>
|
||||
<Paragraph style={{ textAlign: 'center', marginBottom: 24 }}>
|
||||
输入大学官网地址，系统将自动分析网站结构并生成爬虫脚本
|
||||
</Paragraph>
|
||||
|
||||
<Form
|
||||
form={form}
|
||||
layout="vertical"
|
||||
onFinish={handleSubmit}
|
||||
>
|
||||
<Form.Item
|
||||
name="url"
|
||||
label="大学官网URL"
|
||||
rules={[
|
||||
{ required: true, message: '请输入大学官网URL' },
|
||||
{ type: 'url', message: '请输入有效的URL' }
|
||||
]}
|
||||
>
|
||||
<Input
|
||||
placeholder="https://www.harvard.edu/"
|
||||
size="large"
|
||||
prefix={<GlobalOutlined />}
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
name="name"
|
||||
label="大学名称 (可选)"
|
||||
>
|
||||
<Input
|
||||
placeholder="如: Harvard University"
|
||||
size="large"
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item>
|
||||
<Button
|
||||
type="primary"
|
||||
htmlType="submit"
|
||||
size="large"
|
||||
block
|
||||
icon={<RocketOutlined />}
|
||||
>
|
||||
一键生成爬虫脚本
|
||||
</Button>
|
||||
</Form.Item>
|
||||
</Form>
|
||||
|
||||
<div style={{ marginTop: 32, padding: 16, background: '#f5f5f5', borderRadius: 8 }}>
|
||||
<Text strong>支持的大学类型:</Text>
|
||||
<ul style={{ marginTop: 8 }}>
|
||||
<li>美国大学 (如 Harvard, MIT, Stanford)</li>
|
||||
<li>英国大学 (如 Oxford, Cambridge)</li>
|
||||
<li>其他海外大学</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{currentStep === 1 && (
|
||||
<div style={{ textAlign: 'center', padding: 60 }}>
|
||||
<Spin size="large" />
|
||||
<Title level={4} style={{ marginTop: 24 }}>正在分析网站结构...</Title>
|
||||
<Paragraph>系统正在访问大学官网，分析页面结构并生成爬虫脚本</Paragraph>
|
||||
<Paragraph type="secondary">这可能需要几秒钟，请稍候...</Paragraph>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{currentStep === 2 && (
|
||||
<Result
|
||||
status="success"
|
||||
title="爬虫脚本生成成功!"
|
||||
subTitle="系统已自动分析网站结构并生成了爬虫脚本"
|
||||
extra={[
|
||||
<Button
|
||||
type="primary"
|
||||
key="detail"
|
||||
size="large"
|
||||
onClick={() => navigate(`/university/${universityId}`)}
|
||||
>
|
||||
进入大学管理页面
|
||||
</Button>,
|
||||
<Button
|
||||
key="add"
|
||||
size="large"
|
||||
onClick={() => {
|
||||
setCurrentStep(0)
|
||||
form.resetFields()
|
||||
}}
|
||||
>
|
||||
继续添加
|
||||
</Button>
|
||||
]}
|
||||
/>
|
||||
)}
|
||||
</Card>
|
||||
)
|
||||
}
|
||||
185
frontend/src/pages/HomePage.tsx
Normal file
185
frontend/src/pages/HomePage.tsx
Normal file
@ -0,0 +1,185 @@
|
||||
/**
|
||||
* 首页 - 大学列表
|
||||
*/
|
||||
import { useState } from 'react'
|
||||
import { useNavigate } from 'react-router-dom'
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import {
|
||||
Card, Table, Button, Input, Space, Tag, message, Popconfirm, Typography, Row, Col, Statistic
|
||||
} from 'antd'
|
||||
import {
|
||||
PlusOutlined, SearchOutlined, DeleteOutlined, EyeOutlined, ReloadOutlined
|
||||
} from '@ant-design/icons'
|
||||
import { universityApi } from '../services/api'
|
||||
|
||||
const { Title } = Typography
|
||||
|
||||
// 状态标签映射
|
||||
const statusTags: Record<string, { color: string; text: string }> = {
|
||||
pending: { color: 'default', text: '待分析' },
|
||||
analyzing: { color: 'processing', text: '分析中' },
|
||||
ready: { color: 'success', text: '就绪' },
|
||||
error: { color: 'error', text: '错误' }
|
||||
}
|
||||
|
||||
export default function HomePage() {
|
||||
const navigate = useNavigate()
|
||||
const queryClient = useQueryClient()
|
||||
const [search, setSearch] = useState('')
|
||||
|
||||
// 获取大学列表
|
||||
const { data, isLoading, refetch } = useQuery({
|
||||
queryKey: ['universities', search],
|
||||
queryFn: () => universityApi.list({ search: search || undefined })
|
||||
})
|
||||
|
||||
// 删除大学
|
||||
const deleteMutation = useMutation({
|
||||
mutationFn: universityApi.delete,
|
||||
onSuccess: () => {
|
||||
message.success('删除成功')
|
||||
queryClient.invalidateQueries({ queryKey: ['universities'] })
|
||||
},
|
||||
onError: () => {
|
||||
message.error('删除失败')
|
||||
}
|
||||
})
|
||||
|
||||
const universities = data?.data?.items || []
|
||||
const total = data?.data?.total || 0
|
||||
|
||||
// 统计
|
||||
const readyCount = universities.filter((u: any) => u.status === 'ready').length
|
||||
const totalPrograms = universities.reduce((sum: number, u: any) =>
|
||||
sum + (u.latest_result?.programs_count || 0), 0)
|
||||
const totalFaculty = universities.reduce((sum: number, u: any) =>
|
||||
sum + (u.latest_result?.faculty_count || 0), 0)
|
||||
|
||||
const columns = [
|
||||
{
|
||||
title: '大学名称',
|
||||
dataIndex: 'name',
|
||||
key: 'name',
|
||||
render: (text: string, record: any) => (
|
||||
<a onClick={() => navigate(`/university/${record.id}`)}>{text}</a>
|
||||
)
|
||||
},
|
||||
{
|
||||
title: '国家',
|
||||
dataIndex: 'country',
|
||||
key: 'country',
|
||||
width: 100
|
||||
},
|
||||
{
|
||||
title: '状态',
|
||||
dataIndex: 'status',
|
||||
key: 'status',
|
||||
width: 100,
|
||||
render: (status: string) => {
|
||||
const tag = statusTags[status] || { color: 'default', text: status }
|
||||
return <Tag color={tag.color}>{tag.text}</Tag>
|
||||
}
|
||||
},
|
||||
{
|
||||
title: '项目数',
|
||||
key: 'programs',
|
||||
width: 100,
|
||||
render: (_: any, record: any) => record.latest_result?.programs_count || '-'
|
||||
},
|
||||
{
|
||||
title: '导师数',
|
||||
key: 'faculty',
|
||||
width: 100,
|
||||
render: (_: any, record: any) => record.latest_result?.faculty_count || '-'
|
||||
},
|
||||
{
|
||||
title: '操作',
|
||||
key: 'actions',
|
||||
width: 150,
|
||||
render: (_: any, record: any) => (
|
||||
<Space>
|
||||
<Button
|
||||
type="link"
|
||||
icon={<EyeOutlined />}
|
||||
onClick={() => navigate(`/university/${record.id}`)}
|
||||
>
|
||||
查看
|
||||
</Button>
|
||||
<Popconfirm
|
||||
title="确定删除这个大学吗?"
|
||||
onConfirm={() => deleteMutation.mutate(record.id)}
|
||||
okText="确定"
|
||||
cancelText="取消"
|
||||
>
|
||||
<Button type="link" danger icon={<DeleteOutlined />}>
|
||||
删除
|
||||
</Button>
|
||||
</Popconfirm>
|
||||
</Space>
|
||||
)
|
||||
}
|
||||
]
|
||||
|
||||
return (
|
||||
<div>
|
||||
{/* 统计卡片 */}
|
||||
<Row gutter={16} style={{ marginBottom: 24 }}>
|
||||
<Col span={6}>
|
||||
<Card>
|
||||
<Statistic title="大学总数" value={total} />
|
||||
</Card>
|
||||
</Col>
|
||||
<Col span={6}>
|
||||
<Card>
|
||||
<Statistic title="已就绪" value={readyCount} valueStyle={{ color: '#52c41a' }} />
|
||||
</Card>
|
||||
</Col>
|
||||
<Col span={6}>
|
||||
<Card>
|
||||
<Statistic title="项目总数" value={totalPrograms} />
|
||||
</Card>
|
||||
</Col>
|
||||
<Col span={6}>
|
||||
<Card>
|
||||
<Statistic title="导师总数" value={totalFaculty} />
|
||||
</Card>
|
||||
</Col>
|
||||
</Row>
|
||||
|
||||
{/* 大学列表 */}
|
||||
<Card
|
||||
title={<Title level={4} style={{ margin: 0 }}>大学列表</Title>}
|
||||
extra={
|
||||
<Space>
|
||||
<Input
|
||||
placeholder="搜索大学..."
|
||||
prefix={<SearchOutlined />}
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
style={{ width: 200 }}
|
||||
allowClear
|
||||
/>
|
||||
<Button icon={<ReloadOutlined />} onClick={() => refetch()}>
|
||||
刷新
|
||||
</Button>
|
||||
<Button type="primary" icon={<PlusOutlined />} onClick={() => navigate('/add')}>
|
||||
添加大学
|
||||
</Button>
|
||||
</Space>
|
||||
}
|
||||
>
|
||||
<Table
|
||||
columns={columns}
|
||||
dataSource={universities}
|
||||
rowKey="id"
|
||||
loading={isLoading}
|
||||
pagination={{
|
||||
total,
|
||||
showSizeChanger: true,
|
||||
showTotal: (t) => `共 ${t} 所大学`
|
||||
}}
|
||||
/>
|
||||
</Card>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
368
frontend/src/pages/UniversityDetailPage.tsx
Normal file
368
frontend/src/pages/UniversityDetailPage.tsx
Normal file
@ -0,0 +1,368 @@
|
||||
/**
|
||||
* 大学详情页面 - 管理爬虫、运行爬虫、查看数据
|
||||
*/
|
||||
import { useState, useEffect } from 'react'
|
||||
import { useParams, useNavigate } from 'react-router-dom'
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
|
||||
import {
|
||||
Card, Tabs, Button, Typography, Tag, Space, Table, Progress, Timeline, Spin,
|
||||
message, Descriptions, Tree, Input, Row, Col, Statistic, Empty, Modal
|
||||
} from 'antd'
|
||||
import {
|
||||
PlayCircleOutlined, ReloadOutlined, DownloadOutlined, ArrowLeftOutlined,
|
||||
CheckCircleOutlined, ClockCircleOutlined, ExclamationCircleOutlined,
|
||||
SearchOutlined, TeamOutlined, BookOutlined, BankOutlined
|
||||
} from '@ant-design/icons'
|
||||
import { universityApi, scriptApi, jobApi, resultApi } from '../services/api'
|
||||
|
||||
const { Title, Text, Paragraph } = Typography
|
||||
const { TabPane } = Tabs
|
||||
|
||||
// 状态映射
|
||||
const statusMap: Record<string, { color: string; text: string; icon: any }> = {
|
||||
pending: { color: 'default', text: '等待中', icon: <ClockCircleOutlined /> },
|
||||
running: { color: 'processing', text: '运行中', icon: <Spin size="small" /> },
|
||||
completed: { color: 'success', text: '已完成', icon: <CheckCircleOutlined /> },
|
||||
failed: { color: 'error', text: '失败', icon: <ExclamationCircleOutlined /> },
|
||||
cancelled: { color: 'warning', text: '已取消', icon: <ExclamationCircleOutlined /> }
|
||||
}
|
||||
|
||||
export default function UniversityDetailPage() {
|
||||
const { id } = useParams<{ id: string }>()
|
||||
const navigate = useNavigate()
|
||||
const queryClient = useQueryClient()
|
||||
const universityId = parseInt(id || '0')
|
||||
|
||||
const [activeTab, setActiveTab] = useState('overview')
|
||||
const [pollingJobId, setPollingJobId] = useState<number | null>(null)
|
||||
const [searchKeyword, setSearchKeyword] = useState('')
|
||||
|
||||
// 获取大学详情
|
||||
const { data: universityData, isLoading: universityLoading } = useQuery({
|
||||
queryKey: ['university', universityId],
|
||||
queryFn: () => universityApi.get(universityId)
|
||||
})
|
||||
|
||||
// 获取脚本
|
||||
const { data: scriptsData } = useQuery({
|
||||
queryKey: ['scripts', universityId],
|
||||
queryFn: () => scriptApi.getByUniversity(universityId)
|
||||
})
|
||||
|
||||
// 获取任务列表
|
||||
const { data: jobsData, refetch: refetchJobs } = useQuery({
|
||||
queryKey: ['jobs', universityId],
|
||||
queryFn: () => jobApi.getByUniversity(universityId)
|
||||
})
|
||||
|
||||
// 获取结果数据
|
||||
const { data: resultData } = useQuery({
|
||||
queryKey: ['result', universityId],
|
||||
queryFn: () => resultApi.get(universityId),
|
||||
enabled: activeTab === 'data'
|
||||
})
|
||||
|
||||
// 获取任务状态 (轮询)
|
||||
const { data: jobStatusData } = useQuery({
|
||||
queryKey: ['job-status', pollingJobId],
|
||||
queryFn: () => jobApi.getStatus(pollingJobId!),
|
||||
enabled: !!pollingJobId,
|
||||
refetchInterval: pollingJobId ? 2000 : false
|
||||
})
|
||||
|
||||
// 启动爬虫任务
|
||||
const startJobMutation = useMutation({
|
||||
mutationFn: () => jobApi.start(universityId),
|
||||
onSuccess: (response) => {
|
||||
message.success('爬虫任务已启动')
|
||||
setPollingJobId(response.data.id)
|
||||
refetchJobs()
|
||||
},
|
||||
onError: (error: any) => {
|
||||
message.error(error.response?.data?.detail || '启动失败')
|
||||
}
|
||||
})
|
||||
|
||||
// 监听任务完成
|
||||
useEffect(() => {
|
||||
if (jobStatusData?.data?.status === 'completed' || jobStatusData?.data?.status === 'failed') {
|
||||
setPollingJobId(null)
|
||||
refetchJobs()
|
||||
queryClient.invalidateQueries({ queryKey: ['university', universityId] })
|
||||
queryClient.invalidateQueries({ queryKey: ['result', universityId] })
|
||||
|
||||
if (jobStatusData?.data?.status === 'completed') {
|
||||
message.success('爬取完成!')
|
||||
} else {
|
||||
message.error('爬取失败')
|
||||
}
|
||||
}
|
||||
}, [jobStatusData?.data?.status])
|
||||
|
||||
const university = universityData?.data
|
||||
const scripts = scriptsData?.data || []
|
||||
const jobs = jobsData?.data || []
|
||||
const result = resultData?.data
|
||||
|
||||
// 构建数据树
|
||||
const buildDataTree = () => {
|
||||
if (!result?.result_data?.schools) return []
|
||||
|
||||
return result.result_data.schools.map((school: any, si: number) => ({
|
||||
key: `school-${si}`,
|
||||
title: (
|
||||
<span>
|
||||
<BankOutlined style={{ marginRight: 8 }} />
|
||||
{school.name} ({school.programs?.length || 0}个项目)
|
||||
</span>
|
||||
),
|
||||
children: school.programs?.map((prog: any, pi: number) => ({
|
||||
key: `program-${si}-${pi}`,
|
||||
title: (
|
||||
<span>
|
||||
<BookOutlined style={{ marginRight: 8 }} />
|
||||
{prog.name} ({prog.faculty?.length || 0}位导师)
|
||||
</span>
|
||||
),
|
||||
children: prog.faculty?.map((fac: any, fi: number) => ({
|
||||
key: `faculty-${si}-${pi}-${fi}`,
|
||||
title: (
|
||||
<span>
|
||||
<TeamOutlined style={{ marginRight: 8 }} />
|
||||
<a href={fac.url} target="_blank" rel="noreferrer">{fac.name}</a>
|
||||
</span>
|
||||
),
|
||||
isLeaf: true
|
||||
}))
|
||||
}))
|
||||
}))
|
||||
}
|
||||
|
||||
if (universityLoading) {
|
||||
return <Card><Spin size="large" /></Card>
|
||||
}
|
||||
|
||||
if (!university) {
|
||||
return <Card><Empty description="大学不存在" /></Card>
|
||||
}
|
||||
|
||||
const activeScript = scripts.find((s: any) => s.status === 'active')
|
||||
const latestJob = jobs[0]
|
||||
const isRunning = pollingJobId !== null || latestJob?.status === 'running'
|
||||
|
||||
return (
|
||||
<div>
|
||||
{/* 头部 */}
|
||||
<Card style={{ marginBottom: 16 }}>
|
||||
<Space style={{ marginBottom: 16 }}>
|
||||
<Button icon={<ArrowLeftOutlined />} onClick={() => navigate('/')}>
|
||||
返回列表
|
||||
</Button>
|
||||
</Space>
|
||||
|
||||
<Row gutter={24}>
|
||||
<Col span={16}>
|
||||
<Title level={3} style={{ marginBottom: 8 }}>{university.name}</Title>
|
||||
<Paragraph>
|
||||
<a href={university.url} target="_blank" rel="noreferrer">{university.url}</a>
|
||||
</Paragraph>
|
||||
<Space>
|
||||
<Tag>{university.country || '未知国家'}</Tag>
|
||||
<Tag color={university.status === 'ready' ? 'green' : 'orange'}>
|
||||
{university.status === 'ready' ? '就绪' : university.status}
|
||||
</Tag>
|
||||
</Space>
|
||||
</Col>
|
||||
<Col span={8} style={{ textAlign: 'right' }}>
|
||||
<Button
|
||||
type="primary"
|
||||
size="large"
|
||||
icon={isRunning ? <Spin size="small" /> : <PlayCircleOutlined />}
|
||||
onClick={() => startJobMutation.mutate()}
|
||||
disabled={!activeScript || isRunning}
|
||||
loading={startJobMutation.isPending}
|
||||
>
|
||||
{isRunning ? '爬虫运行中...' : '一键运行爬虫'}
|
||||
</Button>
|
||||
</Col>
|
||||
</Row>
|
||||
|
||||
{/* 统计 */}
|
||||
<Row gutter={16} style={{ marginTop: 24 }}>
|
||||
<Col span={6}>
|
||||
<Statistic title="学院数" value={university.latest_result?.schools_count || 0} />
|
||||
</Col>
|
||||
<Col span={6}>
|
||||
<Statistic title="项目数" value={university.latest_result?.programs_count || 0} />
|
||||
</Col>
|
||||
<Col span={6}>
|
||||
<Statistic title="导师数" value={university.latest_result?.faculty_count || 0} />
|
||||
</Col>
|
||||
<Col span={6}>
|
||||
<Statistic title="脚本版本" value={activeScript?.version || 0} />
|
||||
</Col>
|
||||
</Row>
|
||||
</Card>
|
||||
|
||||
{/* 运行进度 */}
|
||||
{pollingJobId && jobStatusData?.data && (
|
||||
<Card style={{ marginBottom: 16 }}>
|
||||
<Title level={5}>爬虫运行中</Title>
|
||||
<Progress percent={jobStatusData.data.progress} status="active" />
|
||||
<Text type="secondary">{jobStatusData.data.current_step}</Text>
|
||||
|
||||
<div style={{ marginTop: 16, maxHeight: 200, overflowY: 'auto' }}>
|
||||
<Timeline
|
||||
items={jobStatusData.data.logs?.slice(-10).map((log: any) => ({
|
||||
color: log.level === 'error' ? 'red' : log.level === 'warning' ? 'orange' : 'blue',
|
||||
children: <Text>{log.message}</Text>
|
||||
}))}
|
||||
/>
|
||||
</div>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{/* 标签页 */}
|
||||
<Card>
|
||||
<Tabs activeKey={activeTab} onChange={setActiveTab}>
|
||||
{/* 概览 */}
|
||||
<TabPane tab="概览" key="overview">
|
||||
<Descriptions title="基本信息" bordered column={2}>
|
||||
<Descriptions.Item label="大学名称">{university.name}</Descriptions.Item>
|
||||
<Descriptions.Item label="官网地址">
|
||||
<a href={university.url} target="_blank" rel="noreferrer">{university.url}</a>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="国家">{university.country || '-'}</Descriptions.Item>
|
||||
<Descriptions.Item label="状态">
|
||||
<Tag color={university.status === 'ready' ? 'green' : 'default'}>
|
||||
{university.status}
|
||||
</Tag>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="创建时间">
|
||||
{new Date(university.created_at).toLocaleString()}
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="更新时间">
|
||||
{new Date(university.updated_at).toLocaleString()}
|
||||
</Descriptions.Item>
|
||||
</Descriptions>
|
||||
|
||||
<Title level={5} style={{ marginTop: 24 }}>历史任务</Title>
|
||||
<Table
|
||||
dataSource={jobs.slice(0, 5)}
|
||||
rowKey="id"
|
||||
pagination={false}
|
||||
columns={[
|
||||
{
|
||||
title: '任务ID',
|
||||
dataIndex: 'id',
|
||||
width: 80
|
||||
},
|
||||
{
|
||||
title: '状态',
|
||||
dataIndex: 'status',
|
||||
width: 100,
|
||||
render: (status: string) => {
|
||||
const s = statusMap[status] || { color: 'default', text: status }
|
||||
return <Tag color={s.color}>{s.icon} {s.text}</Tag>
|
||||
}
|
||||
},
|
||||
{
|
||||
title: '进度',
|
||||
dataIndex: 'progress',
|
||||
width: 150,
|
||||
render: (progress: number) => <Progress percent={progress} size="small" />
|
||||
},
|
||||
{
|
||||
title: '开始时间',
|
||||
dataIndex: 'started_at',
|
||||
render: (t: string) => t ? new Date(t).toLocaleString() : '-'
|
||||
},
|
||||
{
|
||||
title: '完成时间',
|
||||
dataIndex: 'completed_at',
|
||||
render: (t: string) => t ? new Date(t).toLocaleString() : '-'
|
||||
}
|
||||
]}
|
||||
/>
|
||||
</TabPane>
|
||||
|
||||
{/* 数据查看 */}
|
||||
<TabPane tab="数据查看" key="data">
|
||||
{result?.result_data ? (
|
||||
<div>
|
||||
<Row style={{ marginBottom: 16 }}>
|
||||
<Col span={12}>
|
||||
<Input
|
||||
placeholder="搜索项目或导师..."
|
||||
prefix={<SearchOutlined />}
|
||||
value={searchKeyword}
|
||||
onChange={(e) => setSearchKeyword(e.target.value)}
|
||||
style={{ width: 300 }}
|
||||
/>
|
||||
</Col>
|
||||
<Col span={12} style={{ textAlign: 'right' }}>
|
||||
<Button
|
||||
icon={<DownloadOutlined />}
|
||||
onClick={() => {
|
||||
const dataStr = JSON.stringify(result.result_data, null, 2)
|
||||
const blob = new Blob([dataStr], { type: 'application/json' })
|
||||
const url = URL.createObjectURL(blob)
|
||||
const a = document.createElement('a')
|
||||
a.href = url
|
||||
a.download = `${university.name}_data.json`
|
||||
a.click()
|
||||
}}
|
||||
>
|
||||
导出JSON
|
||||
</Button>
|
||||
</Col>
|
||||
</Row>
|
||||
|
||||
<Tree
|
||||
showLine
|
||||
defaultExpandedKeys={['school-0']}
|
||||
treeData={buildDataTree()}
|
||||
style={{ background: '#fafafa', padding: 16, borderRadius: 8 }}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<Empty description="暂无数据，请先运行爬虫" />
|
||||
)}
|
||||
</TabPane>
|
||||
|
||||
{/* 脚本管理 */}
|
||||
<TabPane tab="脚本管理" key="script">
|
||||
{activeScript ? (
|
||||
<div>
|
||||
<Descriptions bordered column={2}>
|
||||
<Descriptions.Item label="脚本名称">{activeScript.script_name}</Descriptions.Item>
|
||||
<Descriptions.Item label="版本">v{activeScript.version}</Descriptions.Item>
|
||||
<Descriptions.Item label="状态">
|
||||
<Tag color="green">活跃</Tag>
|
||||
</Descriptions.Item>
|
||||
<Descriptions.Item label="创建时间">
|
||||
{new Date(activeScript.created_at).toLocaleString()}
|
||||
</Descriptions.Item>
|
||||
</Descriptions>
|
||||
|
||||
<Title level={5} style={{ marginTop: 24 }}>脚本代码</Title>
|
||||
<pre style={{
|
||||
background: '#1e1e1e',
|
||||
color: '#d4d4d4',
|
||||
padding: 16,
|
||||
borderRadius: 8,
|
||||
maxHeight: 400,
|
||||
overflow: 'auto'
|
||||
}}>
|
||||
{activeScript.script_content}
|
||||
</pre>
|
||||
</div>
|
||||
) : (
|
||||
<Empty description="暂无脚本" />
|
||||
)}
|
||||
</TabPane>
|
||||
</Tabs>
|
||||
</Card>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
77
frontend/src/services/api.ts
Normal file
77
frontend/src/services/api.ts
Normal file
@ -0,0 +1,77 @@
|
||||
/**
|
||||
* API服务
|
||||
*/
|
||||
import axios from 'axios'
|
||||
|
||||
const api = axios.create({
|
||||
baseURL: '/api',
|
||||
timeout: 60000
|
||||
})
|
||||
|
||||
// 大学相关API
|
||||
export const universityApi = {
|
||||
list: (params?: { skip?: number; limit?: number; search?: string }) =>
|
||||
api.get('/universities', { params }),
|
||||
|
||||
get: (id: number) =>
|
||||
api.get(`/universities/${id}`),
|
||||
|
||||
create: (data: { name: string; url: string; country?: string }) =>
|
||||
api.post('/universities', data),
|
||||
|
||||
update: (id: number, data: { name?: string; url?: string; country?: string }) =>
|
||||
api.put(`/universities/${id}`, data),
|
||||
|
||||
delete: (id: number) =>
|
||||
api.delete(`/universities/${id}`)
|
||||
}
|
||||
|
||||
// 脚本相关API
|
||||
export const scriptApi = {
|
||||
generate: (data: { university_url: string; university_name?: string }) =>
|
||||
api.post('/scripts/generate', data),
|
||||
|
||||
getByUniversity: (universityId: number) =>
|
||||
api.get(`/scripts/university/${universityId}`),
|
||||
|
||||
get: (id: number) =>
|
||||
api.get(`/scripts/${id}`)
|
||||
}
|
||||
|
||||
// 任务相关API
|
||||
export const jobApi = {
|
||||
start: (universityId: number) =>
|
||||
api.post(`/jobs/start/${universityId}`),
|
||||
|
||||
get: (id: number) =>
|
||||
api.get(`/jobs/${id}`),
|
||||
|
||||
getStatus: (id: number) =>
|
||||
api.get(`/jobs/${id}/status`),
|
||||
|
||||
getByUniversity: (universityId: number) =>
|
||||
api.get(`/jobs/university/${universityId}`),
|
||||
|
||||
cancel: (id: number) =>
|
||||
api.post(`/jobs/${id}/cancel`)
|
||||
}
|
||||
|
||||
// 结果相关API
|
||||
export const resultApi = {
|
||||
get: (universityId: number) =>
|
||||
api.get(`/results/university/${universityId}`),
|
||||
|
||||
getSchools: (universityId: number) =>
|
||||
api.get(`/results/university/${universityId}/schools`),
|
||||
|
||||
getPrograms: (universityId: number, params?: { school_name?: string; search?: string }) =>
|
||||
api.get(`/results/university/${universityId}/programs`, { params }),
|
||||
|
||||
getFaculty: (universityId: number, params?: { school_name?: string; program_name?: string; search?: string; skip?: number; limit?: number }) =>
|
||||
api.get(`/results/university/${universityId}/faculty`, { params }),
|
||||
|
||||
export: (universityId: number) =>
|
||||
api.get(`/results/university/${universityId}/export`, { responseType: 'blob' })
|
||||
}
|
||||
|
||||
export default api
|
||||
1
frontend/src/vite-env.d.ts
vendored
Normal file
1
frontend/src/vite-env.d.ts
vendored
Normal file
@ -0,0 +1 @@
|
||||
/// <reference types="vite/client" />
|
||||
21
frontend/tsconfig.json
Normal file
21
frontend/tsconfig.json
Normal file
@ -0,0 +1,21 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true
|
||||
},
|
||||
"include": ["src"],
|
||||
"references": [{ "path": "./tsconfig.node.json" }]
|
||||
}
|
||||
10
frontend/tsconfig.node.json
Normal file
10
frontend/tsconfig.node.json
Normal file
@ -0,0 +1,10 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"composite": true,
|
||||
"skipLibCheck": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"allowSyntheticDefaultImports": true
|
||||
},
|
||||
"include": ["vite.config.ts"]
|
||||
}
|
||||
15
frontend/vite.config.ts
Normal file
15
frontend/vite.config.ts
Normal file
@ -0,0 +1,15 @@
|
||||
import { defineConfig } from 'vite'
|
||||
import react from '@vitejs/plugin-react'
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
port: 3000,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:8000',
|
||||
changeOrigin: true
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
Reference in New Issue
Block a user