You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
38 lines
937 B
TypeScript
38 lines
937 B
TypeScript
import { CrawlImpl } from '../type';
|
|
|
|
export const jina: CrawlImpl<{ apiKey?: string }> = async (url, params) => {
|
|
const token = params.apiKey ?? process.env.JINA_READER_API_KEY ?? process.env.JINA_API_KEY;
|
|
|
|
try {
|
|
const res = await fetch(`https://r.jina.ai/${url}`, {
|
|
headers: {
|
|
'Accept': 'application/json',
|
|
'Authorization': token ? `Bearer ${token}` : '',
|
|
'x-send-from': 'LobeChat Community',
|
|
},
|
|
});
|
|
|
|
if (res.ok) {
|
|
const json = await res.json();
|
|
if (json.code === 200) {
|
|
const result = json.data;
|
|
return {
|
|
content: result.content,
|
|
contentType: 'text',
|
|
description: result?.description,
|
|
length: result.content.length,
|
|
siteName: result?.siteName,
|
|
title: result?.title,
|
|
url: url,
|
|
};
|
|
}
|
|
|
|
throw json;
|
|
}
|
|
} catch (error) {
|
|
console.error(error);
|
|
}
|
|
|
|
return;
|
|
};
|