refactor(@vben/web-antd): 完成爬虫部分未解决问题

This commit is contained in:
vertoryao 2025-05-18 01:17:03 +08:00
parent 2e3a7806fc
commit 65fa961a7f
4 changed files with 118 additions and 102 deletions

View File

@ -2,7 +2,7 @@ import { requestClient } from '#/api/request';
// 全国公共资源爬虫 // 全国公共资源爬虫
export function runSpider(data: any) { export function runSpider(data: any) {
return requestClient.post(`/spider/run`, { data }); return requestClient.post(`/spider/run`, data);
} }
export function getSpiderStatus() { export function getSpiderStatus() {

View File

@ -10,8 +10,8 @@ export default defineConfig(async () => {
changeOrigin: true, changeOrigin: true,
rewrite: (path) => path.replace(/^\/api/, ''), rewrite: (path) => path.replace(/^\/api/, ''),
// mock代理目标地址 // mock代理目标地址
// target: 'http://localhost:8081/api', target: 'http://localhost:8081/api',
target: 'http://43.139.10.64:8082/api', // target: 'http://43.139.10.64:8082/api',
// target: 'http://192.168.3.238:8081/api', // target: 'http://192.168.3.238:8081/api',
ws: true, ws: true,
}, },
@ -20,11 +20,21 @@ export default defineConfig(async () => {
rewrite: (path) => path.replace(/^\/docx/, ''), rewrite: (path) => path.replace(/^\/docx/, ''),
target: 'http://47.112.173.8:6805/static', target: 'http://47.112.173.8:6805/static',
}, },
'/guangzhou': {
changeOrigin: true,
rewrite: (path) => path.replace(/^\/guangzhou/, ''),
target: 'http://47.112.173.8:6800/',
},
'/pptx': { '/pptx': {
changeOrigin: true, changeOrigin: true,
rewrite: (path) => path.replace(/^\/pptx/, ''), rewrite: (path) => path.replace(/^\/pptx/, ''),
target: 'http://47.112.173.8:6802/static', target: 'http://47.112.173.8:6802/static',
}, },
'/spider': {
changeOrigin: true,
rewrite: (path) => path.replace(/^\/spider/, ''),
target: 'http://47.112.173.8:6806/static',
},
'/v1': { '/v1': {
changeOrigin: true, changeOrigin: true,
rewrite: (path) => path.replace(/^\/v1/, ''), rewrite: (path) => path.replace(/^\/v1/, ''),

View File

@ -36,7 +36,8 @@ const [Drawer, drawerApi] = useVbenDrawer({
const data = drawerApi.getData<Record<string, any>>(); const data = drawerApi.getData<Record<string, any>>();
if (data) { if (data) {
isLoading.value = false; // isLoading.value = false; //
docx.value = `/docx/${data}`; // docx // docx.value = `/spider/${data}`; // docx
docx.value = data; // docx
} }
// url.value = drawerApi.getData<Record<string, any>>(); // url.value = drawerApi.getData<Record<string, any>>();
} }

View File

@ -130,6 +130,87 @@ function extractDocxFilename(url: string): null | string {
const match = url.match(/\/static\/([^/]+\.docx)$/i); // /static/ .docx const match = url.match(/\/static\/([^/]+\.docx)$/i); // /static/ .docx
return match ? match[1] : null; return match ? match[1] : null;
} }
const startChecking = () => {
statusPollingInterval = window.setInterval(startPolling, 1000);
};
let statusPollingInterval: null | number;
let pollingCount = 0;
const maxPollingAttempts = 300; // 300
const startPolling = async () => {
try {
const response = await props.getSpiderStatus();
if (!response.data.is_running) {
clearInterval(statusPollingInterval!);
statusPollingInterval = null;
if (response.data.download_url) {
const download_url = response.data.download_url;
fetchResult.value = download_url;
notification.success({ message: '获取成功', duration: 3 });
let filename = '';
if (isDocxURL(download_url)) {
filename = extractDocxFilename(download_url);
}
fetchResult.value = `/spider/${filename}`;
resultItems.value.push({
key: resultItems.value.length + 1,
role: 'ai',
content: '文档已生成',
footer: h(Flex, null, [
h(
Button,
{
size: 'normal',
type: 'primary',
onClick: () => {
openPreviewDrawer('right', fetchResult);
},
},
'文档预览',
),
h(
Button,
{
size: 'normal',
type: 'primary',
style: { marginLeft: '10px' },
onClick: () => {
const link = document.createElement('a');
link.href = download_url;
link.download = fetchResult.value;
document.body.append(link);
link.click();
link.remove();
},
},
'文档下载',
),
]),
});
isFetching.value = false;
fetchStatus.value = 'completed';
} else {
isFetching.value = false;
fetchResult.value = '';
message.error('抓取无结果');
}
} else if (++pollingCount >= maxPollingAttempts) {
clearInterval(statusPollingInterval!);
statusPollingInterval = null;
props.stopSpider();
message.warn('轮询超时,请稍后再试');
}
} catch (error) {
clearInterval(statusPollingInterval!);
statusPollingInterval = null;
message.error(`状态查询失败:${error}`);
}
};
const startFetching = async () => { const startFetching = async () => {
// //
@ -149,7 +230,7 @@ const startFetching = async () => {
try { try {
notification.info({ notification.info({
message: '正在获取中...', message: '正在获取中...',
duration: 3, duration: 5,
}); });
const res = await props.runSpider({ const res = await props.runSpider({
@ -158,94 +239,19 @@ const startFetching = async () => {
llm_api_key: props.item.id, llm_api_key: props.item.id,
}); });
if (res.code !== 0) { if (res.code === 0) {
notification.info({
message: res.msg,
duration: 5,
});
startChecking();
} else {
isFetching.value = false; isFetching.value = false;
fetchResult.value = ''; fetchResult.value = '';
message.error('抓取无结果'); message.error('抓取无结果');
return; return;
} }
let statusPollingInterval: null | number = null;
let pollingCount = 0;
const maxPollingAttempts = 300; // 300
const startPolling = async () => {
try {
const response = await props.getSpiderStatus();
if (!response.data.is_running) {
clearInterval(statusPollingInterval!);
statusPollingInterval = null;
if (response.data.download_url) {
const download_url = response.data.download_url;
fetchResult.value = download_url;
notification.success({ message: '获取成功', duration: 3 });
let filename = '';
if (isDocxURL(download_url)) {
filename = extractDocxFilename(download_url);
}
fetchResult.value = `/static/${filename}`;
resultItems.value.push({
key: resultItems.value.length + 1,
role: 'ai',
content: '文档已生成',
footer: h(Flex, null, [
h(
Button,
{
size: 'normal',
type: 'primary',
onClick: () => {
openPreviewDrawer('right', safeFilename);
},
},
'文档预览',
),
h(
Button,
{
size: 'normal',
type: 'primary',
style: { marginLeft: '10px' },
onClick: () => {
const link = document.createElement('a');
link.href = download_url;
link.download = safeFilename;
document.body.append(link);
link.click();
link.remove();
},
},
'文档下载',
),
]),
});
isFetching.value = false;
fetchStatus.value = 'completed';
} else {
isFetching.value = false;
fetchResult.value = '';
message.error('抓取无结果');
}
} else if (++pollingCount >= maxPollingAttempts) {
clearInterval(statusPollingInterval!);
statusPollingInterval = null;
props.stopSpider();
message.warn('轮询超时,请稍后再试');
}
} catch (error) {
clearInterval(statusPollingInterval!);
statusPollingInterval = null;
message.error(`状态查询失败:${error}`);
}
};
statusPollingInterval = window.setInterval(startPolling, 1000);
// onUnmounted // onUnmounted
// //
// onUnmounted(() => { // onUnmounted(() => {
@ -288,25 +294,24 @@ const startFetching = async () => {
}); });
const fileUrl = ref(''); const fileUrl = ref('');
fileUrl.value = `http://47.112.173.8:6800${fetchResult.value}`; fileUrl.value = `/guangzhou${fetchResult.value}`;
// url http://47.112.173.8:6802/static/66f3cfd95e364a239d8036390db658ae.docx
resultItems.value.push({ resultItems.value.push({
key: resultItems.value.length + 1, key: resultItems.value.length + 1,
role: 'ai', role: 'ai',
content: '文档已生成', content: '文档已生成有效时间5分钟请及时查看或下载',
footer: h(Flex, null, [ footer: h(Flex, null, [
// h( h(
// Button, Button,
// { {
// size: 'nomarl', size: 'nomarl',
// type: 'primary', type: 'primary',
// onClick: () => { onClick: () => {
// openPreviewDrawer('right', fetchResult); openPreviewDrawer('right', fileUrl);
// }, },
// }, },
// '', '文档预览',
// ), ),
h( h(
Button, Button,
{ {
@ -318,7 +323,7 @@ const startFetching = async () => {
onClick: () => { onClick: () => {
// <a> // <a>
const link = document.createElement('a'); const link = document.createElement('a');
link.href = fileUrl; // link.href = fileUrl.value; //
link.download = '广州公共资源交易中心数据获取'; // link.download = '广州公共资源交易中心数据获取'; //
document.body.append(link); // <a> document.body.append(link); // <a>
link.click(); // link.click(); //