vertoryao/vue-vben-admin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(@vben/common-ui): 优化数据抓取

Browse Source
This commit is contained in:
Kven 2025-05-17 22:07:52 +08:00
parent e4a74a5f8c
commit 2e3a7806fc
4 changed files with 314 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/web-antd/src/api/core/spider.ts
View File

@ -6,13 +6,13 @@ export function runSpider(data: any) {
}
export function getSpiderStatus() {
return requestClient.get(`/spider/status`);
return requestClient.post(`/spider/status`);
}
export function getSpiderLogs() {
return requestClient.get(`/v1/workflow/list`);
return requestClient.post(`/v1/workflow/list`);
}
export function stopSpider() {
return requestClient.get(`/spider/stop`);
return requestClient.post(`/spider/stop`);
}

12
apps/web-antd/src/views/spider/index.vue
View File

@ -5,7 +5,14 @@ import { SpiderListView, SpiderWorkView } from '@vben/common-ui';
import { notification } from 'ant-design-vue';
import { getWorkflowInfo, getWorkflowList, runSpider } from '#/api';
import {
getSpiderStatus,
getWorkflowInfo,
getWorkflowList,
runSpider,
sendWorkflow,
stopSpider,
} from '#/api';
// sendWorkflow
interface TempItem {
id: string;
@ -87,6 +94,9 @@ async function handleClickMode(item: TempItem) {
title="目标网址："
:item="temp"
:run-spider="runSpider"
:stop-spider="stopSpider"
:get-spider-status="getSpiderStatus"
:send-workflow="sendWorkflow"
:item-message="itemMessage"
/>
</div>

4
packages/effects/common-ui/src/ui/ppt/ppt-work-view.vue
View File

@ -17,7 +17,7 @@ import {
import { Badge, Button, Flex, Space, Typography } from 'ant-design-vue';
import { Attachments, Bubble, Sender, Welcome } from 'ant-design-x-vue';
import WordPreview from '../word/word-preview.vue';
import PptPreview from './ppt-perview.vue';
interface ResultItem {
key: number;
@ -140,7 +140,7 @@ const fetchResult = ref('');
const [PreviewDrawer, previewDrawerApi] = useVbenDrawer({
// 连接抽离的组件
connectedComponent: WordPreview,
connectedComponent: PptPreview,
// placement: 'left',
});
function openPreviewDrawer(

318
packages/effects/common-ui/src/ui/spider/spider-work-view.vue
View File

@ -37,7 +37,7 @@ interface SpiderParams {
interface SpiderContext {
userId: string;
conversationId: string;
files: unknown[];
files: [];
inputs: Record<string, unknown>;
}
@ -60,10 +60,14 @@ interface Props {
itemMessage?: ResultItem;
item?: SpiderItem;
title: string;
runSpider?: (
params: SpiderParams,
context: SpiderContext,
) => Promise<SpiderResult>;
runSpider?: (context: any) => Promise<SpiderResult>;
getSpiderStatus?: () => Promise<{
status: string;
}>;
stopSpider?: () => Promise<{
status: string;
}>;
sendWorkflow?: (params: SpiderParams, data: SpiderContext) => Promise<any>;
}
defineOptions({
@ -76,10 +80,21 @@ const props = withDefaults(defineProps<Props>(), {
return null;
},
runSpider: () => async () => ({
data: {
msg: '',
code: '',
}),
getSpiderStatus: () => async () => ({
msg: '',
code: '',
}),
stopSpider: () => async () => ({
msg: '',
code: '',
}),
sendWorkflow: () => async () => ({
outputs: {
result: '',
},
files: [],
},
}),
});
@ -112,11 +127,8 @@ function isDocxURL(str: string): boolean {
// 使用正则提取 /static/ 后面的文件名部分
function extractDocxFilename(url: string): null | string {
const match = url.match(/\/static\/([a-f0-9-]+\.docx)$/i);
if (match) {
return match[1]; // 返回类似：9e1c421e-991c-411f-8176-6350a97e70f3.docx
}
return null;
const match = url.match(/\/static\/([^/]+\.docx)$/i); // 匹配 /static/ 后任意非斜杠字符加 .docx
return match ? match[1] : null;
}
const startFetching = async () => {
@ -133,48 +145,49 @@ const startFetching = async () => {
publish_end_time =
dayjs(selectedDateRange.value[1]).format('YYYYMMDDhhmmss') || '';
}
if (props.item.id === '77c068fd-d5b6-4c33-97d8-db5511a09b26') {
try {
notification.info({
message: '正在获取中...',
duration: 3,
});
const res = await props.runSpider(
{
const res = await props.runSpider({
publish_start_time,
publish_end_time,
llm_api_key: '77c068fd-d5b6-4c33-97d8-db5511a09b26',
},
// {
// appid: props.item.id,
// },
// {
// userId: '1562',
// conversationId: '',
// files: [],
// inputs: {
// publish_start_time,
// publish_end_time,
// },
// },
);
if (res.data.outputs.result) {
// 保存抓取结果
fetchResult.value = res.data.outputs.result;
notification.success({
message: '获取成功',
duration: 3,
llm_api_key: props.item.id,
});
const { result } = res.data.outputs;
const filename = ref('');
if (result && isDocxURL(result)) {
filename.value = extractDocxFilename(result);
if (res.code !== 0) {
isFetching.value = false;
fetchResult.value = '';
message.error('抓取无结果');
return;
}
// 保存抓取结果 url http://47.112.173.8:6802/static/66f3cfd95e364a239d8036390db658ae.docx
fetchResult.value = `/static/${filename.value}`;
let statusPollingInterval: null | number = null;
let pollingCount = 0;
const maxPollingAttempts = 300; // 最多轮询300秒
const startPolling = async () => {
try {
const response = await props.getSpiderStatus();
if (!response.data.is_running) {
clearInterval(statusPollingInterval!);
statusPollingInterval = null;
if (response.data.download_url) {
const download_url = response.data.download_url;
fetchResult.value = download_url;
notification.success({ message: '获取成功', duration: 3 });
let filename = '';
if (isDocxURL(download_url)) {
filename = extractDocxFilename(download_url);
}
fetchResult.value = `/static/${filename}`;
resultItems.value.push({
key: resultItems.value.length + 1,
role: 'ai',
@ -183,14 +196,117 @@ const startFetching = async () => {
h(
Button,
{
size: 'nomarl',
size: 'normal',
type: 'primary',
onClick: () => {
openPreviewDrawer('right', filename);
openPreviewDrawer('right', safeFilename);
},
},
'文档预览',
),
h(
Button,
{
size: 'normal',
type: 'primary',
style: { marginLeft: '10px' },
onClick: () => {
const link = document.createElement('a');
link.href = download_url;
link.download = safeFilename;
document.body.append(link);
link.click();
link.remove();
},
},
'文档下载',
),
]),
});
isFetching.value = false;
fetchStatus.value = 'completed';
} else {
isFetching.value = false;
fetchResult.value = '';
message.error('抓取无结果');
}
} else if (++pollingCount >= maxPollingAttempts) {
clearInterval(statusPollingInterval!);
statusPollingInterval = null;
props.stopSpider();
message.warn('轮询超时，请稍后再试');
}
} catch (error) {
clearInterval(statusPollingInterval!);
statusPollingInterval = null;
message.error(`状态查询失败：${error}`);
}
};
statusPollingInterval = window.setInterval(startPolling, 1000);
// 组件卸载时清理定时器（需配合 onUnmounted 生命周期）
// 示例：
// onUnmounted(() => {
// if (statusPollingInterval) {
// clearInterval(statusPollingInterval);
// }
// });
} catch (error) {
isFetching.value = false;
fetchResult.value = '';
console.error(error);
}
}
if (props.item.id === 'c736edd0-925d-4877-9223-56aab7342311') {
try {
notification.info({
message: '正在获取中...',
duration: 3,
});
const res = await props.sendWorkflow(
{
appid: props.item.id,
},
{
userId: '1562',
conversationId: '',
files: [],
inputs: {
publish_start_time,
publish_end_time,
},
},
);
if (res.data.outputs.files) {
// 保存抓取结果
fetchResult.value = res.data.outputs.files[0].url;
notification.success({
message: '获取成功',
duration: 3,
});
const fileUrl = ref('');
fileUrl.value = `http://47.112.173.8:6800${fetchResult.value}`;
// 保存抓取结果 url http://47.112.173.8:6802/static/66f3cfd95e364a239d8036390db658ae.docx
resultItems.value.push({
key: resultItems.value.length + 1,
role: 'ai',
content: '文档已生成',
footer: h(Flex, null, [
// h(
// Button,
// {
// size: 'nomarl',
// type: 'primary',
// onClick: () => {
// openPreviewDrawer('right', fetchResult);
// },
// },
// '文档预览',
// ),
h(
Button,
{
@ -202,8 +318,8 @@ const startFetching = async () => {
onClick: () => {
// 创建隐藏的 <a> 标签用于触发下载
const link = document.createElement('a');
link.href = msg.content; // 设置下载链接
link.download = filename; // 设置下载文件名
link.href = fileUrl; // 设置下载链接
link.download = '广州公共资源交易中心数据获取'; // 设置下载文件名
document.body.append(link); // 将 <a> 标签添加到页面中
link.click(); // 触发点击事件开始下载
link.remove(); // 下载完成后移除 <a> 标签
@ -221,9 +337,84 @@ const startFetching = async () => {
} catch (error) {
message.error(`${error}`);
}
// 模拟抓取完成
isFetching.value = false;
}
// try {
// notification.info({
// message: '正在获取中...',
// duration: 3,
// });
// const res = await props.runSpider(
// {
// publish_start_time,
// publish_end_time,
// llm_api_key: '77c068fd-d5b6-4c33-97d8-db5511a09b26',
// },
// );
// if (res.data.outputs.result) {
// // 保存抓取结果
// fetchResult.value = res.data.outputs.result;
// notification.success({
// message: '获取成功',
// duration: 3,
// });
// const { result } = res.data.outputs;
//
// const filename = ref('');
//
// if (result && isDocxURL(result)) {
// filename.value = extractDocxFilename(result);
// }
//
// // 保存抓取结果 url http://47.112.173.8:6802/static/66f3cfd95e364a239d8036390db658ae.docx
// fetchResult.value = `/static/${filename.value}`;
// resultItems.value.push({
// key: resultItems.value.length + 1,
// role: 'ai',
// content: '文档已生成',
// footer: h(Flex, null, [
// h(
// Button,
// {
// size: 'nomarl',
// type: 'primary',
// onClick: () => {
// openPreviewDrawer('right', filename);
// },
// },
// '文档预览',
// ),
// h(
// Button,
// {
// size: 'normal',
// type: 'primary',
// style: {
// marginLeft: '10px',
// },
// onClick: () => {
// // 创建隐藏的 <a> 标签用于触发下载
// const link = document.createElement('a');
// link.href = msg.content; // 设置下载链接
// link.download = filename; // 设置下载文件名
// document.body.append(link); // 将 <a> 标签添加到页面中
// link.click(); // 触发点击事件开始下载
// link.remove(); // 下载完成后移除 <a> 标签
// },
// },
// '文档下载',
// ),
// ]),
// });
// fetchStatus.value = 'completed';
// } else {
// fetchResult.value = '';
// message.error('抓取无结果');
// }
// } catch (error) {
// message.error(`${error}`);
// }
};
// 列表角色
@ -287,27 +478,22 @@ watch(
footer: msg.footer,
});
} else {
const filename = ref('');
if (msg.content && isDocxURL(msg.content)) {
filename.value = extractDocxFilename(msg.content);
}
resultItems.value.push({
key: resultItems.value.length + 1,
role: msg.role, // 'user' or 'ai'
content: '文档已生成',
footer: h(Flex, null, [
h(
Button,
{
size: 'normal',
type: 'primary',
onClick: () => {
openPreviewDrawer('right', filename);
},
},
'文档预览',
),
// h(
// Button,
// {
// size: 'normal',
// type: 'primary',
// onClick: () => {
// openPreviewDrawer('right', filename);
// },
// },
// '文档预览',
// ),
h(
Button,
{
@ -320,7 +506,7 @@ watch(
// 创建隐藏的 <a> 标签用于触发下载
const link = document.createElement('a');
link.href = msg.content; // 设置下载链接
link.download = filename; // 设置下载文件名
link.download = '广州公共资源交易中心数据获取'; // 设置下载文件名
document.body.append(link); // 将 <a> 标签添加到页面中
link.click(); // 触发点击事件开始下载
link.remove(); // 下载完成后移除 <a> 标签