Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0f03ad9801 | |||
| 3eef54496c |
@@ -1,3 +1,18 @@
|
|||||||
|
# weibo-rss
|
||||||
|
|
||||||
|
> 基于原作者v0.3.0代码修改
|
||||||
|
|
||||||
|
#### v0.3.0-2
|
||||||
|
1. 新增页码URL参数`?page=1-3`、`?page=1,3,5`(需cookie;支持混用,如`?page=1-3,10,8`)
|
||||||
|
|
||||||
|
#### v0.3.0-1
|
||||||
|
1. 新增视频/Live图链接以适配日记簿朋友圈
|
||||||
|
2. 图片链接不再转URL编码及百度代理
|
||||||
|
3. 新增自定义cookie环境变量`WEIBO_COOKIE=SUB=...`
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# weibo-rss
|
# weibo-rss
|
||||||
|
|
||||||
简单的微博 RSS 订阅源生成器,可将某人最近发布的微博转换为符合 RSS Feed 标准的格式,供阅读器订阅。
|
简单的微博 RSS 订阅源生成器,可将某人最近发布的微博转换为符合 RSS Feed 标准的格式,供阅读器订阅。
|
||||||
|
|||||||
@@ -7,3 +7,5 @@ services:
|
|||||||
restart: always
|
restart: always
|
||||||
ports:
|
ports:
|
||||||
- "3000:3000"
|
- "3000:3000"
|
||||||
|
environment:
|
||||||
|
- WEIBO_COOKIE=${WEIBO_COOKIE:-}
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "weibo-rss",
|
"name": "weibo-rss",
|
||||||
"version": "0.3.0",
|
"version": "0.3.0-1",
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "tsc",
|
"build": "tsc",
|
||||||
|
|||||||
+6
-1
@@ -52,6 +52,10 @@
|
|||||||
<button class="btn btn-default">生成</button>
|
<button class="btn btn-default">生成</button>
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="input-group" style="margin-top: 10px;">
|
||||||
|
<span class="input-group-addon">页码</span>
|
||||||
|
<input id="page-input" type="text" class="form-control" value="1" placeholder="1 / 7 / 1-10 / 1,3,5">
|
||||||
|
</div>
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
<div class="loading" style="display: none;"><p class="text-center">加载中...</p></div>
|
<div class="loading" style="display: none;"><p class="text-center">加载中...</p></div>
|
||||||
@@ -123,7 +127,8 @@ function showResult(uid) {
|
|||||||
}
|
}
|
||||||
var baseUrl = location.href.split("?")[0];
|
var baseUrl = location.href.split("?")[0];
|
||||||
if (baseUrl.substr(-1, 1) !== "/") baseUrl += "/";
|
if (baseUrl.substr(-1, 1) !== "/") baseUrl += "/";
|
||||||
var feedUrl = baseUrl + "rss/user/" + uid;
|
var page = $.trim($("#page-input").val()) || "1";
|
||||||
|
var feedUrl = baseUrl + "rss/user/" + uid + "?page=" + encodeURIComponent(page);
|
||||||
|
|
||||||
$("#foo").val(feedUrl);
|
$("#foo").val(feedUrl);
|
||||||
for (var reader in prefix) {
|
for (var reader in prefix) {
|
||||||
|
|||||||
+2
-2
@@ -31,8 +31,8 @@ const defaultConfig = {
|
|||||||
apiDetail: 7 * 24 * 60 * 60,
|
apiDetail: 7 * 24 * 60 * 60,
|
||||||
apiDomain: 7 * 24 * 60 * 60,
|
apiDomain: 7 * 24 * 60 * 60,
|
||||||
},
|
},
|
||||||
// 图片缓存代理
|
// 微博 Cookie,可通过环境变量 WEIBO_COOKIE 或 config.js 指定
|
||||||
imageCache: 'https://image.baidu.com/search/down?url=',
|
weiboCookie: process.env.WEIBO_COOKIE || '',
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
import { describe, expect, test } from "@jest/globals";
|
||||||
|
import { parsePageSpec, PageInvalidError } from "./page";
|
||||||
|
|
||||||
|
describe('page spec parser', () => {
|
||||||
|
test('defaults to first page', () => {
|
||||||
|
expect(parsePageSpec(undefined)).toEqual({
|
||||||
|
pages: [1],
|
||||||
|
key: '1',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('parses a single page', () => {
|
||||||
|
expect(parsePageSpec('7')).toEqual({
|
||||||
|
pages: [7],
|
||||||
|
key: '7',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('parses a page range', () => {
|
||||||
|
expect(parsePageSpec('1-3')).toEqual({
|
||||||
|
pages: [1, 2, 3],
|
||||||
|
key: '1,2,3',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('parses a page list', () => {
|
||||||
|
expect(parsePageSpec('1,3,5')).toEqual({
|
||||||
|
pages: [1, 3, 5],
|
||||||
|
key: '1,3,5',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('parses a mixed page spec', () => {
|
||||||
|
expect(parsePageSpec('1-3, 5')).toEqual({
|
||||||
|
pages: [1, 2, 3, 5],
|
||||||
|
key: '1,2,3,5',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
test('rejects invalid page input', () => {
|
||||||
|
expect(() => parsePageSpec('0')).toThrow(PageInvalidError);
|
||||||
|
expect(() => parsePageSpec('3-1')).toThrow(PageInvalidError);
|
||||||
|
expect(() => parsePageSpec('abc')).toThrow(PageInvalidError);
|
||||||
|
expect(() => parsePageSpec('1,,3')).toThrow(PageInvalidError);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
export class PageInvalidError extends Error {
|
||||||
|
constructor(page: string) {
|
||||||
|
super(`page: ${page}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PageSpec {
|
||||||
|
pages: number[];
|
||||||
|
key: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const SINGLE_PAGE_PATTERN = /^[1-9]\d*$/;
|
||||||
|
const PAGE_RANGE_PATTERN = /^([1-9]\d*)\s*-\s*([1-9]\d*)$/;
|
||||||
|
|
||||||
|
const normalizePages = (pages: number[]) => {
|
||||||
|
const seen = new Set<number>();
|
||||||
|
const normalized: number[] = [];
|
||||||
|
|
||||||
|
for (const page of pages) {
|
||||||
|
if (seen.has(page)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
seen.add(page);
|
||||||
|
normalized.push(page);
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalized;
|
||||||
|
};
|
||||||
|
|
||||||
|
const parsePageToken = (token: string) => {
|
||||||
|
if (SINGLE_PAGE_PATTERN.test(token)) {
|
||||||
|
return [Number(token)];
|
||||||
|
}
|
||||||
|
|
||||||
|
const rangeMatch = token.match(PAGE_RANGE_PATTERN);
|
||||||
|
if (rangeMatch) {
|
||||||
|
const start = Number(rangeMatch[1]);
|
||||||
|
const end = Number(rangeMatch[2]);
|
||||||
|
if (end < start) {
|
||||||
|
throw new PageInvalidError(token);
|
||||||
|
}
|
||||||
|
|
||||||
|
return Array.from({ length: end - start + 1 }, (_, index) => start + index);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new PageInvalidError(token);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const parsePageSpec = (page: unknown): PageSpec => {
|
||||||
|
if (page === undefined || page === null) {
|
||||||
|
return {
|
||||||
|
pages: [1],
|
||||||
|
key: '1',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Array.isArray(page)) {
|
||||||
|
throw new PageInvalidError(page.join(','));
|
||||||
|
}
|
||||||
|
|
||||||
|
const rawPage = String(page).trim();
|
||||||
|
if (!rawPage) {
|
||||||
|
return {
|
||||||
|
pages: [1],
|
||||||
|
key: '1',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const tokens = rawPage.split(',');
|
||||||
|
if (tokens.some(token => !token.trim())) {
|
||||||
|
throw new PageInvalidError(rawPage);
|
||||||
|
}
|
||||||
|
|
||||||
|
const pages: number[] = [];
|
||||||
|
for (const token of tokens) {
|
||||||
|
const tokenPages = parsePageToken(token.trim());
|
||||||
|
pages.push(...tokenPages);
|
||||||
|
}
|
||||||
|
|
||||||
|
const normalizedPages = normalizePages(pages);
|
||||||
|
if (!normalizedPages.length) {
|
||||||
|
throw new PageInvalidError(rawPage);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
pages: normalizedPages,
|
||||||
|
key: normalizedPages.join(','),
|
||||||
|
};
|
||||||
|
};
|
||||||
+24
-3
@@ -5,9 +5,10 @@ import Router from '@koa/router';
|
|||||||
import NodeRSS from 'rss';
|
import NodeRSS from 'rss';
|
||||||
import { RSSKoaContext, RSSKoaState } from '../types';
|
import { RSSKoaContext, RSSKoaState } from '../types';
|
||||||
import config from '../config';
|
import config from '../config';
|
||||||
import { DomainNotFoundError, statusToHTML, UserNotFoundError } from './weibo/weibo';
|
import { DomainNotFoundError, statusToHTML, UserNotFoundError, WeiboCookieInvalidError } from './weibo/weibo';
|
||||||
import { ThrottledError } from './throttler';
|
import { ThrottledError } from './throttler';
|
||||||
import { logger } from './logger';
|
import { logger } from './logger';
|
||||||
|
import { PageInvalidError, parsePageSpec } from './page';
|
||||||
|
|
||||||
export class UidInvalidError extends Error {
|
export class UidInvalidError extends Error {
|
||||||
constructor(uid: string) {
|
constructor(uid: string) {
|
||||||
@@ -32,10 +33,12 @@ export const registerRoutes = (
|
|||||||
throw new UidInvalidError(uid);
|
throw new UidInvalidError(uid);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const pageSpec = parsePageSpec(ctx.query.page);
|
||||||
|
|
||||||
// get data
|
// get data
|
||||||
let cacheMiss = false;
|
let cacheMiss = false;
|
||||||
const xmlData = await ctx.cache.memo(async () => {
|
const xmlData = await ctx.cache.memo(async () => {
|
||||||
const weiboData = await ctx.weibo.fetchUserLatestWeibo(uid);
|
const weiboData = await ctx.weibo.fetchUserLatestWeibo(uid, pageSpec.pages);
|
||||||
if (weiboData) {
|
if (weiboData) {
|
||||||
// basic info
|
// basic info
|
||||||
const feed = new NodeRSS({
|
const feed = new NodeRSS({
|
||||||
@@ -59,7 +62,7 @@ export const registerRoutes = (
|
|||||||
cacheMiss = true;
|
cacheMiss = true;
|
||||||
return feed.xml();
|
return feed.xml();
|
||||||
}
|
}
|
||||||
}, `xml-${uid}`, config.cacheTTL.rssXml);
|
}, `xml-${uid}-page-${pageSpec.key}`, config.cacheTTL.rssXml);
|
||||||
|
|
||||||
// send data
|
// send data
|
||||||
ctx.set('Content-Type', 'text/xml');
|
ctx.set('Content-Type', 'text/xml');
|
||||||
@@ -78,6 +81,16 @@ export const registerRoutes = (
|
|||||||
ctx.body = `找不到用户,可能用户仅登录可见,不支持订阅。可以通过打开 https://m.weibo.cn/u/:uid 验证(<a href="https://m.weibo.cn/u/${uid}" target="_blank">uid: ${uid}</a>)`;
|
ctx.body = `找不到用户,可能用户仅登录可见,不支持订阅。可以通过打开 https://m.weibo.cn/u/:uid 验证(<a href="https://m.weibo.cn/u/${uid}" target="_blank">uid: ${uid}</a>)`;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (error instanceof PageInvalidError) {
|
||||||
|
ctx.status = 400;
|
||||||
|
ctx.body = `page 参数格式有误。支持单页如 7,范围如 1-10,或列表如 1,3,5。page: ${error.message.replace(/^page:\s*/, '')}`;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (error instanceof WeiboCookieInvalidError) {
|
||||||
|
ctx.status = 401;
|
||||||
|
ctx.body = `微博 Cookie 可能已过期或失效,请更新 WEIBO_COOKIE 后重试。uid: ${uid}`;
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (error instanceof ThrottledError) {
|
if (error instanceof ThrottledError) {
|
||||||
ctx.status = 503;
|
ctx.status = 503;
|
||||||
ctx.body = `暂时无法拉取到数据,请稍后再试。uid: ${uid}`;
|
ctx.body = `暂时无法拉取到数据,请稍后再试。uid: ${uid}`;
|
||||||
@@ -123,6 +136,14 @@ export const registerRoutes = (
|
|||||||
};
|
};
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (error instanceof WeiboCookieInvalidError) {
|
||||||
|
ctx.status = 401;
|
||||||
|
ctx.body = {
|
||||||
|
success: false,
|
||||||
|
msg: '微博 Cookie 可能已过期或失效,请更新 WEIBO_COOKIE 后重试',
|
||||||
|
};
|
||||||
|
return;
|
||||||
|
}
|
||||||
logger.error(error);
|
logger.error(error);
|
||||||
ctx.status = 500;
|
ctx.status = 500;
|
||||||
ctx.body = {
|
ctx.body = {
|
||||||
|
|||||||
@@ -1,10 +1,51 @@
|
|||||||
import { AxiosError } from "axios";
|
import { AxiosError } from "axios";
|
||||||
|
import config from "../../../config";
|
||||||
|
|
||||||
export const TIME_OUT = 3000;
|
export const TIME_OUT = 3000;
|
||||||
export const MOCK_UA = 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36';
|
export const MOCK_UA = 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36';
|
||||||
|
|
||||||
|
export class WeiboCookieInvalidError extends Error {
|
||||||
|
constructor() {
|
||||||
|
super('Weibo cookie is invalid or expired');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const getWeiboCookie = () => (config.weiboCookie || '').trim();
|
||||||
|
|
||||||
|
export const getWeiboCookieHeaders = () => {
|
||||||
|
const cookie = getWeiboCookie();
|
||||||
|
return cookie ? { Cookie: cookie } : {};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const hasWeiboCookie = () => getWeiboCookie().length > 0;
|
||||||
|
|
||||||
|
const hasLoginErrorMessage = (data: any) => {
|
||||||
|
const msg = [
|
||||||
|
data?.msg,
|
||||||
|
data?.message,
|
||||||
|
data?.error,
|
||||||
|
data?.data?.msg,
|
||||||
|
data?.data?.message,
|
||||||
|
].filter(Boolean).join(' ');
|
||||||
|
return /登录|登陆|cookie|身份|认证|授权|过期|失效|login|expired|invalid|unauthorized/i.test(msg);
|
||||||
|
};
|
||||||
|
|
||||||
|
const hasLoginErrorCode = (data: any) => {
|
||||||
|
const code = String(data?.errno || data?.errcode || data?.code || data?.error_code || '');
|
||||||
|
return ['100005', '100006', '20003', '20019', '401', '403', '418'].includes(code);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const throwIfWeiboCookieInvalid = (data: any) => {
|
||||||
|
if (hasWeiboCookie() && (hasLoginErrorMessage(data) || hasLoginErrorCode(data))) {
|
||||||
|
throw new WeiboCookieInvalidError();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
export const handleForbiddenErr = (err: AxiosError, cb: () => Promise<void>) => {
|
export const handleForbiddenErr = (err: AxiosError, cb: () => Promise<void>) => {
|
||||||
if (err.response && [418, 403].includes(err.response.status)) {
|
if (err.response && [401, 403, 418].includes(err.response.status)) {
|
||||||
|
if (hasWeiboCookie()) {
|
||||||
|
return Promise.reject(new WeiboCookieInvalidError());
|
||||||
|
}
|
||||||
return cb();
|
return cb();
|
||||||
} else {
|
} else {
|
||||||
return Promise.reject(err);
|
return Promise.reject(err);
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { Throttler } from "../../throttler";
|
import { Throttler } from "../../throttler";
|
||||||
import Axios from "axios";
|
import Axios from "axios";
|
||||||
import { Agent } from "https";
|
import { Agent } from "https";
|
||||||
import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
|
import { getWeiboCookieHeaders, handleForbiddenErr, MOCK_UA, throwIfWeiboCookieInvalid, TIME_OUT } from "./common";
|
||||||
import { logger } from "../../logger";
|
import { logger } from "../../logger";
|
||||||
import { waitMs } from "../../../utils";
|
import { waitMs } from "../../../utils";
|
||||||
|
|
||||||
@@ -10,7 +10,8 @@ export const createDetailAPI = () => {
|
|||||||
const httpsAgent = new Agent({ keepAlive: true });
|
const httpsAgent = new Agent({ keepAlive: true });
|
||||||
const axiosInstance = Axios.create({
|
const axiosInstance = Axios.create({
|
||||||
timeout: TIME_OUT,
|
timeout: TIME_OUT,
|
||||||
httpsAgent
|
httpsAgent,
|
||||||
|
headers: getWeiboCookieHeaders(),
|
||||||
});
|
});
|
||||||
return {
|
return {
|
||||||
getWeiboDetail: (id: string) => runner.runFunc(async (disable) => {
|
getWeiboDetail: (id: string) => runner.runFunc(async (disable) => {
|
||||||
@@ -26,6 +27,7 @@ export const createDetailAPI = () => {
|
|||||||
'X-Requested-With': 'XMLHttpRequest'
|
'X-Requested-With': 'XMLHttpRequest'
|
||||||
},
|
},
|
||||||
}).then(({ data }) => {
|
}).then(({ data }) => {
|
||||||
|
throwIfWeiboCookieInvalid(data);
|
||||||
return data.data;
|
return data.data;
|
||||||
}).catch(err => handleForbiddenErr(err, disable));
|
}).catch(err => handleForbiddenErr(err, disable));
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { Throttler } from "../../throttler";
|
import { Throttler } from "../../throttler";
|
||||||
import Axios from "axios";
|
import Axios from "axios";
|
||||||
import { Agent } from "https";
|
import { Agent } from "https";
|
||||||
import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
|
import { getWeiboCookieHeaders, handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
|
||||||
import { logger } from "../../logger";
|
import { logger } from "../../logger";
|
||||||
import { waitMs } from "../../../utils";
|
import { waitMs } from "../../../utils";
|
||||||
|
|
||||||
@@ -16,7 +16,8 @@ export const createDomainAPI = () => {
|
|||||||
const httpsAgent = new Agent({ keepAlive: true });
|
const httpsAgent = new Agent({ keepAlive: true });
|
||||||
const axiosInstance = Axios.create({
|
const axiosInstance = Axios.create({
|
||||||
timeout: TIME_OUT,
|
timeout: TIME_OUT,
|
||||||
httpsAgent
|
httpsAgent,
|
||||||
|
headers: getWeiboCookieHeaders(),
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { Throttler } from "../../throttler";
|
import { Throttler } from "../../throttler";
|
||||||
import Axios from "axios";
|
import Axios from "axios";
|
||||||
import { Agent } from "https";
|
import { Agent } from "https";
|
||||||
import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
|
import { getWeiboCookie, getWeiboCookieHeaders, handleForbiddenErr, MOCK_UA, throwIfWeiboCookieInvalid, TIME_OUT } from "./common";
|
||||||
import { WeiboStatus, WeiboUserData } from "../../../types";
|
import { WeiboStatus, WeiboUserData } from "../../../types";
|
||||||
import { logger } from "../../logger";
|
import { logger } from "../../logger";
|
||||||
import { waitMs } from "../../../utils";
|
import { waitMs } from "../../../utils";
|
||||||
@@ -17,11 +17,13 @@ export const createIndexAPI = () => {
|
|||||||
const httpsAgent = new Agent({ keepAlive: true });
|
const httpsAgent = new Agent({ keepAlive: true });
|
||||||
const axiosInstance = Axios.create({
|
const axiosInstance = Axios.create({
|
||||||
timeout: TIME_OUT,
|
timeout: TIME_OUT,
|
||||||
httpsAgent
|
httpsAgent,
|
||||||
|
headers: getWeiboCookieHeaders(),
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
getIndexUserInfo: (uid: string) => runner.runFunc<WeiboUserData>(async (disable) => {
|
getIndexUserInfo: (uid: string) => runner.runFunc<WeiboUserData>(async (disable) => {
|
||||||
|
if (!getWeiboCookie()) {
|
||||||
const resHeaders = await axiosInstance({
|
const resHeaders = await axiosInstance({
|
||||||
"method": "POST",
|
"method": "POST",
|
||||||
"url": "https://visitor.passport.weibo.cn/visitor/genvisitor2",
|
"url": "https://visitor.passport.weibo.cn/visitor/genvisitor2",
|
||||||
@@ -36,6 +38,7 @@ export const createIndexAPI = () => {
|
|||||||
}
|
}
|
||||||
}).then(res => res.headers)
|
}).then(res => res.headers)
|
||||||
axiosInstance.defaults.headers.common['Cookie'] = resHeaders['set-cookie'].filter(cookie => cookie.startsWith('SUB')).map(cookie => cookie.split(';')[0]).join(';');
|
axiosInstance.defaults.headers.common['Cookie'] = resHeaders['set-cookie'].filter(cookie => cookie.startsWith('SUB')).map(cookie => cookie.split(';')[0]).join(';');
|
||||||
|
}
|
||||||
logger.debug(`[getInfo] ${uid}`);
|
logger.debug(`[getInfo] ${uid}`);
|
||||||
await waitMs(Math.floor(Math.random() * 100));
|
await waitMs(Math.floor(Math.random() * 100));
|
||||||
return await axiosInstance({
|
return await axiosInstance({
|
||||||
@@ -48,6 +51,7 @@ export const createIndexAPI = () => {
|
|||||||
'X-Requested-With': 'XMLHttpRequest'
|
'X-Requested-With': 'XMLHttpRequest'
|
||||||
}
|
}
|
||||||
}).then(({ data }) => {
|
}).then(({ data }) => {
|
||||||
|
throwIfWeiboCookieInvalid(data);
|
||||||
if (data.ok !== 1) {
|
if (data.ok !== 1) {
|
||||||
return Promise.reject(new UserNotFoundError(uid));
|
return Promise.reject(new UserNotFoundError(uid));
|
||||||
}
|
}
|
||||||
@@ -60,12 +64,12 @@ export const createIndexAPI = () => {
|
|||||||
return result;
|
return result;
|
||||||
}).catch(err => handleForbiddenErr(err, disable));
|
}).catch(err => handleForbiddenErr(err, disable));
|
||||||
}),
|
}),
|
||||||
getWeiboContentList: (uid: string, containerId: string) => runner.runFunc<WeiboStatus[]>(async (disable) => {
|
getWeiboContentList: (uid: string, containerId: string, page: number = 1) => runner.runFunc<WeiboStatus[]>(async (disable) => {
|
||||||
logger.debug(`[getContList] ${uid} ${containerId}`);
|
logger.debug(`[getContList] ${uid} ${containerId} page=${page}`);
|
||||||
await waitMs(Math.floor(Math.random() * 100));
|
await waitMs(Math.floor(Math.random() * 100));
|
||||||
return await axiosInstance({
|
return await axiosInstance({
|
||||||
method: 'get',
|
method: 'get',
|
||||||
url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}&containerid=${containerId}`,
|
url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}&containerid=${containerId}&page=${page}`,
|
||||||
headers: {
|
headers: {
|
||||||
'Mweibo-Pwa': 1,
|
'Mweibo-Pwa': 1,
|
||||||
'Referer': `https://m.weibo.cn/u/${uid}`,
|
'Referer': `https://m.weibo.cn/u/${uid}`,
|
||||||
@@ -73,6 +77,7 @@ export const createIndexAPI = () => {
|
|||||||
'X-Requested-With': 'XMLHttpRequest'
|
'X-Requested-With': 'XMLHttpRequest'
|
||||||
}
|
}
|
||||||
}).then(({ data }) => {
|
}).then(({ data }) => {
|
||||||
|
throwIfWeiboCookieInvalid(data);
|
||||||
return data.data.cards
|
return data.data.cards
|
||||||
.filter(item => item.mblog)
|
.filter(item => item.mblog)
|
||||||
.map(item => item.mblog);
|
.map(item => item.mblog);
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { Throttler } from "../../throttler";
|
import { Throttler } from "../../throttler";
|
||||||
import Axios from "axios";
|
import Axios from "axios";
|
||||||
import { Agent } from "https";
|
import { Agent } from "https";
|
||||||
import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
|
import { getWeiboCookieHeaders, handleForbiddenErr, MOCK_UA, throwIfWeiboCookieInvalid, TIME_OUT } from "./common";
|
||||||
import { logger } from "../../logger";
|
import { logger } from "../../logger";
|
||||||
import { waitMs } from "../../../utils";
|
import { waitMs } from "../../../utils";
|
||||||
|
|
||||||
@@ -10,7 +10,8 @@ export const createLongTextAPI = () => {
|
|||||||
const httpsAgent = new Agent({ keepAlive: true });
|
const httpsAgent = new Agent({ keepAlive: true });
|
||||||
const axiosInstance = Axios.create({
|
const axiosInstance = Axios.create({
|
||||||
timeout: TIME_OUT,
|
timeout: TIME_OUT,
|
||||||
httpsAgent
|
httpsAgent,
|
||||||
|
headers: getWeiboCookieHeaders(),
|
||||||
});
|
});
|
||||||
return {
|
return {
|
||||||
getWeiboLongText: (id: string) => runner.runFunc<string>(async (disable) => {
|
getWeiboLongText: (id: string) => runner.runFunc<string>(async (disable) => {
|
||||||
@@ -26,6 +27,7 @@ export const createLongTextAPI = () => {
|
|||||||
'X-Requested-With': 'XMLHttpRequest'
|
'X-Requested-With': 'XMLHttpRequest'
|
||||||
},
|
},
|
||||||
}).then(({ data }) => {
|
}).then(({ data }) => {
|
||||||
|
throwIfWeiboCookieInvalid(data);
|
||||||
if (!data.data) {
|
if (!data.data) {
|
||||||
throw new Error(JSON.stringify(data));
|
throw new Error(JSON.stringify(data));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, test } from '@jest/globals';
|
import { describe, expect, test } from '@jest/globals';
|
||||||
import { WeiboData } from "./weibo";
|
import { statusToHTML, WeiboData } from "./weibo";
|
||||||
import { WeiboStatus } from '../../types';
|
import { WeiboStatus } from '../../types';
|
||||||
|
|
||||||
const wbData = new WeiboData({
|
const wbData = new WeiboData({
|
||||||
@@ -36,3 +36,33 @@ describe('Weibo Data: domain to uid', () => {
|
|||||||
expect(resData).toBe('1197161814');
|
expect(resData).toBe('1197161814');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('statusToHTML', () => {
|
||||||
|
test('add video poster tag and hide weibo video text', () => {
|
||||||
|
const videoUrl = 'https://video.weibo.com/show?fid=1034:1234567890&raw=//video';
|
||||||
|
const coverUrl = 'https://wx1.sinaimg.cn/orj480/cover.jpg?raw=//cover';
|
||||||
|
const shortUrl = 'http://t.cn/A6K3ITkN?foo=1&bar=//raw';
|
||||||
|
const videoPosterPattern = /<video[^>]*?poster\s*=\s*(['"])(.*?)\1[^>]*?>/i;
|
||||||
|
const html = statusToHTML({
|
||||||
|
text: '正文<a href="https://m.weibo.cn/p/index"><span class="surl-text">搜狐新闻的微博视频</span></a>',
|
||||||
|
page_info: {
|
||||||
|
type: 'video',
|
||||||
|
page_url: videoUrl,
|
||||||
|
url_ori: shortUrl,
|
||||||
|
page_pic: {
|
||||||
|
url: coverUrl,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as WeiboStatus);
|
||||||
|
|
||||||
|
expect(html).not.toContain('搜狐新闻的微博视频');
|
||||||
|
expect(html).toContain('<video src="' + videoUrl + '" poster="' + coverUrl + '"></video>');
|
||||||
|
expect(html.match(videoPosterPattern)?.[2]).toBe(coverUrl);
|
||||||
|
expect(html).not.toContain(shortUrl);
|
||||||
|
expect(html).not.toContain('视频链接');
|
||||||
|
expect(html).not.toContain('视频封面');
|
||||||
|
expect(html).not.toContain('<img src="' + coverUrl + '">');
|
||||||
|
expect(html).not.toContain(encodeURIComponent(videoUrl));
|
||||||
|
expect(html).not.toContain(encodeURIComponent(coverUrl));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -5,10 +5,12 @@ import { createDetailAPI, GetWeiboDetailFunc } from "./api/detailAPI";
|
|||||||
import { createDomainAPI, DomainNotFoundError, GetUIDByDomainFunc } from "./api/domainAPI";
|
import { createDomainAPI, DomainNotFoundError, GetUIDByDomainFunc } from "./api/domainAPI";
|
||||||
import { createIndexAPI, GetIndexUserInfoFunc, GetWeiboContentListFunc, UserNotFoundError } from "./api/indexAPI";
|
import { createIndexAPI, GetIndexUserInfoFunc, GetWeiboContentListFunc, UserNotFoundError } from "./api/indexAPI";
|
||||||
import { createLongTextAPI, GetWeiboLongTextFunc } from "./api/longTextAPI";
|
import { createLongTextAPI, GetWeiboLongTextFunc } from "./api/longTextAPI";
|
||||||
|
import { WeiboCookieInvalidError } from "./api/common";
|
||||||
|
|
||||||
export {
|
export {
|
||||||
DomainNotFoundError,
|
DomainNotFoundError,
|
||||||
UserNotFoundError,
|
UserNotFoundError,
|
||||||
|
WeiboCookieInvalidError,
|
||||||
};
|
};
|
||||||
|
|
||||||
export class WeiboData {
|
export class WeiboData {
|
||||||
@@ -40,15 +42,30 @@ export class WeiboData {
|
|||||||
/**
|
/**
|
||||||
* get user's weibo
|
* get user's weibo
|
||||||
*/
|
*/
|
||||||
fetchUserLatestWeibo = async (uid: string) => {
|
fetchUserLatestWeibo = async (uid: string, pages: number[] = [1]) => {
|
||||||
const indexInfo = await this.cache.memo(() => this.getIndexUserInfo(uid), `info-${uid}`, config.cacheTTL.apiIndexInfo);
|
const indexInfo = await this.cache.memo(() => this.getIndexUserInfo(uid), `info-${uid}`, config.cacheTTL.apiIndexInfo);
|
||||||
const { containerId } = indexInfo;
|
const { containerId } = indexInfo;
|
||||||
const statusList = await this.cache.memo(async () => {
|
const statusList: WeiboStatus[] = [];
|
||||||
const wbList = await this.getWeiboContentList(uid, containerId);
|
const seenStatusIds = new Set<string>();
|
||||||
return await Promise.all(
|
|
||||||
wbList.map(status => this.fillStatusWithLongText(status))
|
for (const page of pages) {
|
||||||
|
const wbList = await this.cache.memo(
|
||||||
|
() => this.getWeiboContentList(uid, containerId, page),
|
||||||
|
`list-${uid}-page-${page}`,
|
||||||
|
config.cacheTTL.apiStatusList,
|
||||||
);
|
);
|
||||||
}, `list-${uid}`, config.cacheTTL.apiStatusList);
|
const freshList = wbList.filter((status) => {
|
||||||
|
if (seenStatusIds.has(status.id)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
seenStatusIds.add(status.id);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
const pageStatusList = await Promise.all(
|
||||||
|
freshList.map(status => this.fillStatusWithLongText(status))
|
||||||
|
);
|
||||||
|
statusList.push(...pageStatusList);
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
...indexInfo,
|
...indexInfo,
|
||||||
@@ -102,12 +119,32 @@ export class WeiboData {
|
|||||||
fetchUIDByDomain = async (domain: string) => this.getUIDByDomain(domain);
|
fetchUIDByDomain = async (domain: string) => this.getUIDByDomain(domain);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const firstString = (...values: any[]): string | undefined => values.find(value => typeof value === 'string' && value.length > 0);
|
||||||
|
|
||||||
|
const getVideoUrl = (pageInfo: WeiboStatus['page_info']) => firstString(
|
||||||
|
pageInfo?.page_url,
|
||||||
|
pageInfo?.media_info?.h5_url,
|
||||||
|
pageInfo?.media_info?.mp4_hd_url,
|
||||||
|
pageInfo?.media_info?.mp4_sd_url,
|
||||||
|
pageInfo?.media_info?.stream_url_hd,
|
||||||
|
pageInfo?.media_info?.stream_url,
|
||||||
|
);
|
||||||
|
|
||||||
|
const getVideoCoverUrl = (pageInfo: WeiboStatus['page_info']) => firstString(
|
||||||
|
pageInfo?.page_pic?.url,
|
||||||
|
pageInfo?.media_info?.cover_image_phone,
|
||||||
|
pageInfo?.media_info?.cover_image,
|
||||||
|
pageInfo?.media_info?.poster,
|
||||||
|
);
|
||||||
|
|
||||||
export const statusToHTML = (status: WeiboStatus) => {
|
export const statusToHTML = (status: WeiboStatus) => {
|
||||||
let tempHTML = status.text;
|
let tempHTML = status.text;
|
||||||
// 表情转文字
|
// 表情转文字
|
||||||
tempHTML = tempHTML.replace(/<span class="url-icon"><img alt="?(.*?)"? src=".*?" style="width:1em; height:1em;".*?\/><\/span>/g, '$1');
|
tempHTML = tempHTML.replace(/<span class="url-icon"><img alt="?(.*?)"? src=".*?" style="width:1em; height:1em;".*?\/><\/span>/g, '$1');
|
||||||
// 去掉外链图标
|
// 去掉外链图标
|
||||||
tempHTML = tempHTML.replace(/<span class='url-icon'><img.*?><\/span>/g, '');
|
tempHTML = tempHTML.replace(/<span class='url-icon'><img.*?><\/span>/g, '');
|
||||||
|
// 去掉微博视频短链文字
|
||||||
|
tempHTML = tempHTML.replace(/<span\s+class=(["'])surl-text\1>[^<]*?的微博视频<\/span>/g, '');
|
||||||
|
|
||||||
// 转发的微博
|
// 转发的微博
|
||||||
if (status.retweeted_status) {
|
if (status.retweeted_status) {
|
||||||
@@ -125,11 +162,21 @@ export const statusToHTML = (status: WeiboStatus) => {
|
|||||||
if (status.pics) {
|
if (status.pics) {
|
||||||
status.pics.forEach(function (item) {
|
status.pics.forEach(function (item) {
|
||||||
tempHTML += "<br><br>";
|
tempHTML += "<br><br>";
|
||||||
const url = config.imageCache ? (config.imageCache + encodeURIComponent(item.large.url)) : item.large.url;
|
const url = item.large.url;
|
||||||
const largeUrl = config.imageCache ? (config.imageCache + encodeURIComponent(item.large.url)) : item.large.url;
|
const largeUrl = item.large.url;
|
||||||
tempHTML += '<a href="' + largeUrl + '" target="_blank"><img src="' + url+ '"></a>';
|
tempHTML += '<a href="' + largeUrl + '" target="_blank"><img src="' + url+ '"></a>';
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 微博视频
|
||||||
|
if (status.page_info?.type === 'video') {
|
||||||
|
const videoUrl = getVideoUrl(status.page_info);
|
||||||
|
const videoCoverUrl = getVideoCoverUrl(status.page_info);
|
||||||
|
if (videoCoverUrl) {
|
||||||
|
const videoSrc = videoUrl ? ' src="' + videoUrl + '"' : '';
|
||||||
|
tempHTML += '<video' + videoSrc + ' poster="' + videoCoverUrl + '"></video>';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return tempHTML;
|
return tempHTML;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,8 +56,22 @@ export interface WeiboStatus {
|
|||||||
// video and other data
|
// video and other data
|
||||||
page_info?: {
|
page_info?: {
|
||||||
type: 'video' | 'search_topic';
|
type: 'video' | 'search_topic';
|
||||||
|
// eg: 'https://video.weibo.com/show?fid=...'
|
||||||
|
page_url?: string;
|
||||||
// eg: 'http://t.cn/A6K3ITkN'
|
// eg: 'http://t.cn/A6K3ITkN'
|
||||||
url_ori: string;
|
url_ori: string;
|
||||||
|
page_pic?: {
|
||||||
|
url: string;
|
||||||
|
[x: string]: any;
|
||||||
|
};
|
||||||
|
media_info?: {
|
||||||
|
h5_url?: string;
|
||||||
|
mp4_hd_url?: string;
|
||||||
|
mp4_sd_url?: string;
|
||||||
|
stream_url?: string;
|
||||||
|
stream_url_hd?: string;
|
||||||
|
[x: string]: any;
|
||||||
|
};
|
||||||
// eg: '搜狐新闻的微博视频'
|
// eg: '搜狐新闻的微博视频'
|
||||||
page_title: string;
|
page_title: string;
|
||||||
// eg: '2022触动瞬间'
|
// eg: '2022触动瞬间'
|
||||||
|
|||||||
Reference in New Issue
Block a user