diff --git a/README.md b/README.md index e610201..7cff49a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,15 @@ +# weibo-rss + +> 基于原作者v0.3.0代码修改 + +#### v0.3.0-1 +1. 新增视频/Live图链接以适配日记簿朋友圈 +2. 图片链接不再转URL编码及百度代理 +3. 新增自定义cookie环境变量`WEIBO_COOKIE=SUB=...` + + + + # weibo-rss 简单的微博 RSS 订阅源生成器，可将某人最近发布的微博转换为符合 RSS Feed 标准的格式，供阅读器订阅。 diff --git a/docker-compose.yml b/docker-compose.yml index 1154214..fa92bee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,3 +7,5 @@ services: restart: always ports: - "3000:3000" + environment: + - WEIBO_COOKIE=${WEIBO_COOKIE:-} diff --git a/package.json b/package.json index 25094e0..7feee9b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "weibo-rss", - "version": "0.3.0", + "version": "0.3.0-1", "private": true, "scripts": { "build": "tsc", diff --git a/src/config.ts b/src/config.ts index ae0d722..a355316 100644 --- a/src/config.ts +++ b/src/config.ts @@ -31,8 +31,8 @@ const defaultConfig = { apiDetail: 7 * 24 * 60 * 60, apiDomain: 7 * 24 * 60 * 60, }, - // 图片缓存代理 - imageCache: 'https://image.baidu.com/search/down?url=', + // 微博 Cookie，可通过环境变量 WEIBO_COOKIE 或 config.js 指定 + weiboCookie: process.env.WEIBO_COOKIE || '', }; /** diff --git a/src/modules/routes.ts b/src/modules/routes.ts index cf2fb65..7af095e 100644 --- a/src/modules/routes.ts +++ b/src/modules/routes.ts @@ -5,7 +5,7 @@ import Router from '@koa/router'; import NodeRSS from 'rss'; import { RSSKoaContext, RSSKoaState } from '../types'; import config from '../config'; -import { DomainNotFoundError, statusToHTML, UserNotFoundError } from './weibo/weibo'; +import { DomainNotFoundError, statusToHTML, UserNotFoundError, WeiboCookieInvalidError } from './weibo/weibo'; import { ThrottledError } from './throttler'; import { logger } from './logger'; @@ -78,6 +78,11 @@ export const registerRoutes = ( ctx.body = `找不到用户，可能用户仅登录可见，不支持订阅。可以通过打开 https://m.weibo.cn/u/:uid 验证（uid: ${uid}）`; return; } + if (error instanceof WeiboCookieInvalidError) { + ctx.status = 401; + ctx.body = `微博 Cookie 可能已过期或失效，请更新 WEIBO_COOKIE 后重试。uid: ${uid}`; + return; + } if (error instanceof ThrottledError) { ctx.status = 503; ctx.body = `暂时无法拉取到数据，请稍后再试。uid: ${uid}`; @@ -123,6 +128,14 @@ export const registerRoutes = ( }; return; } + if (error instanceof WeiboCookieInvalidError) { + ctx.status = 401; + ctx.body = { + success: false, + msg: '微博 Cookie 可能已过期或失效，请更新 WEIBO_COOKIE 后重试', + }; + return; + } logger.error(error); ctx.status = 500; ctx.body = { diff --git a/src/modules/weibo/api/common.ts b/src/modules/weibo/api/common.ts index 16da4c2..4983489 100644 --- a/src/modules/weibo/api/common.ts +++ b/src/modules/weibo/api/common.ts @@ -1,10 +1,51 @@ import { AxiosError } from "axios"; +import config from "../../../config"; export const TIME_OUT = 3000; export const MOCK_UA = 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36'; +export class WeiboCookieInvalidError extends Error { + constructor() { + super('Weibo cookie is invalid or expired'); + } +} + +export const getWeiboCookie = () => (config.weiboCookie || '').trim(); + +export const getWeiboCookieHeaders = () => { + const cookie = getWeiboCookie(); + return cookie ? { Cookie: cookie } : {}; +}; + +export const hasWeiboCookie = () => getWeiboCookie().length > 0; + +const hasLoginErrorMessage = (data: any) => { + const msg = [ + data?.msg, + data?.message, + data?.error, + data?.data?.msg, + data?.data?.message, + ].filter(Boolean).join(' '); + return /登录|登陆|cookie|身份|认证|授权|过期|失效|login|expired|invalid|unauthorized/i.test(msg); +}; + +const hasLoginErrorCode = (data: any) => { + const code = String(data?.errno || data?.errcode || data?.code || data?.error_code || ''); + return ['100005', '100006', '20003', '20019', '401', '403', '418'].includes(code); +}; + +export const throwIfWeiboCookieInvalid = (data: any) => { + if (hasWeiboCookie() && (hasLoginErrorMessage(data) || hasLoginErrorCode(data))) { + throw new WeiboCookieInvalidError(); + } +}; + export const handleForbiddenErr = (err: AxiosError, cb: () => Promise) => { - if (err.response && [418, 403].includes(err.response.status)) { + if (err.response && [401, 403, 418].includes(err.response.status)) { + if (hasWeiboCookie()) { + return Promise.reject(new WeiboCookieInvalidError()); + } return cb(); } else { return Promise.reject(err); diff --git a/src/modules/weibo/api/detailAPI.ts b/src/modules/weibo/api/detailAPI.ts index 943a5cf..2687c07 100644 --- a/src/modules/weibo/api/detailAPI.ts +++ b/src/modules/weibo/api/detailAPI.ts @@ -1,7 +1,7 @@ import { Throttler } from "../../throttler"; import Axios from "axios"; import { Agent } from "https"; -import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common"; +import { getWeiboCookieHeaders, handleForbiddenErr, MOCK_UA, throwIfWeiboCookieInvalid, TIME_OUT } from "./common"; import { logger } from "../../logger"; import { waitMs } from "../../../utils"; @@ -10,7 +10,8 @@ export const createDetailAPI = () => { const httpsAgent = new Agent({ keepAlive: true }); const axiosInstance = Axios.create({ timeout: TIME_OUT, - httpsAgent + httpsAgent, + headers: getWeiboCookieHeaders(), }); return { getWeiboDetail: (id: string) => runner.runFunc(async (disable) => { @@ -26,6 +27,7 @@ export const createDetailAPI = () => { 'X-Requested-With': 'XMLHttpRequest' }, }).then(({ data }) => { + throwIfWeiboCookieInvalid(data); return data.data; }).catch(err => handleForbiddenErr(err, disable)); }), diff --git a/src/modules/weibo/api/domainAPI.ts b/src/modules/weibo/api/domainAPI.ts index 513527c..8477d23 100644 --- a/src/modules/weibo/api/domainAPI.ts +++ b/src/modules/weibo/api/domainAPI.ts @@ -1,7 +1,7 @@ import { Throttler } from "../../throttler"; import Axios from "axios"; import { Agent } from "https"; -import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common"; +import { getWeiboCookieHeaders, handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common"; import { logger } from "../../logger"; import { waitMs } from "../../../utils"; @@ -16,7 +16,8 @@ export const createDomainAPI = () => { const httpsAgent = new Agent({ keepAlive: true }); const axiosInstance = Axios.create({ timeout: TIME_OUT, - httpsAgent + httpsAgent, + headers: getWeiboCookieHeaders(), }); return { diff --git a/src/modules/weibo/api/indexAPI.ts b/src/modules/weibo/api/indexAPI.ts index d83e90b..b857531 100644 --- a/src/modules/weibo/api/indexAPI.ts +++ b/src/modules/weibo/api/indexAPI.ts @@ -1,7 +1,7 @@ import { Throttler } from "../../throttler"; import Axios from "axios"; import { Agent } from "https"; -import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common"; +import { getWeiboCookie, getWeiboCookieHeaders, handleForbiddenErr, MOCK_UA, throwIfWeiboCookieInvalid, TIME_OUT } from "./common"; import { WeiboStatus, WeiboUserData } from "../../../types"; import { logger } from "../../logger"; import { waitMs } from "../../../utils"; @@ -17,25 +17,28 @@ export const createIndexAPI = () => { const httpsAgent = new Agent({ keepAlive: true }); const axiosInstance = Axios.create({ timeout: TIME_OUT, - httpsAgent + httpsAgent, + headers: getWeiboCookieHeaders(), }); return { getIndexUserInfo: (uid: string) => runner.runFunc(async (disable) => { - const resHeaders = await axiosInstance({ - "method": "POST", - "url": "https://visitor.passport.weibo.cn/visitor/genvisitor2", - "headers": { - "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": MOCK_UA, - }, - "data": { - "cb": "visitor_gray_callback", - tid: '', - from: 'weibo' - } - }).then(res => res.headers) - axiosInstance.defaults.headers.common['Cookie'] = resHeaders['set-cookie'].filter(cookie => cookie.startsWith('SUB')).map(cookie => cookie.split(';')[0]).join(';'); + if (!getWeiboCookie()) { + const resHeaders = await axiosInstance({ + "method": "POST", + "url": "https://visitor.passport.weibo.cn/visitor/genvisitor2", + "headers": { + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": MOCK_UA, + }, + "data": { + "cb": "visitor_gray_callback", + tid: '', + from: 'weibo' + } + }).then(res => res.headers) + axiosInstance.defaults.headers.common['Cookie'] = resHeaders['set-cookie'].filter(cookie => cookie.startsWith('SUB')).map(cookie => cookie.split(';')[0]).join(';'); + } logger.debug(`[getInfo] ${uid}`); await waitMs(Math.floor(Math.random() * 100)); return await axiosInstance({ @@ -48,6 +51,7 @@ export const createIndexAPI = () => { 'X-Requested-With': 'XMLHttpRequest' } }).then(({ data }) => { + throwIfWeiboCookieInvalid(data); if (data.ok !== 1) { return Promise.reject(new UserNotFoundError(uid)); } @@ -73,6 +77,7 @@ export const createIndexAPI = () => { 'X-Requested-With': 'XMLHttpRequest' } }).then(({ data }) => { + throwIfWeiboCookieInvalid(data); return data.data.cards .filter(item => item.mblog) .map(item => item.mblog); diff --git a/src/modules/weibo/api/longTextAPI.ts b/src/modules/weibo/api/longTextAPI.ts index e32ffad..8ee6127 100644 --- a/src/modules/weibo/api/longTextAPI.ts +++ b/src/modules/weibo/api/longTextAPI.ts @@ -1,7 +1,7 @@ import { Throttler } from "../../throttler"; import Axios from "axios"; import { Agent } from "https"; -import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common"; +import { getWeiboCookieHeaders, handleForbiddenErr, MOCK_UA, throwIfWeiboCookieInvalid, TIME_OUT } from "./common"; import { logger } from "../../logger"; import { waitMs } from "../../../utils"; @@ -10,7 +10,8 @@ export const createLongTextAPI = () => { const httpsAgent = new Agent({ keepAlive: true }); const axiosInstance = Axios.create({ timeout: TIME_OUT, - httpsAgent + httpsAgent, + headers: getWeiboCookieHeaders(), }); return { getWeiboLongText: (id: string) => runner.runFunc(async (disable) => { @@ -26,6 +27,7 @@ export const createLongTextAPI = () => { 'X-Requested-With': 'XMLHttpRequest' }, }).then(({ data }) => { + throwIfWeiboCookieInvalid(data); if (!data.data) { throw new Error(JSON.stringify(data)); } diff --git a/src/modules/weibo/weibo.spec.ts b/src/modules/weibo/weibo.spec.ts index f5bbf55..8b289ee 100644 --- a/src/modules/weibo/weibo.spec.ts +++ b/src/modules/weibo/weibo.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from '@jest/globals'; -import { WeiboData } from "./weibo"; +import { statusToHTML, WeiboData } from "./weibo"; import { WeiboStatus } from '../../types'; const wbData = new WeiboData({ @@ -36,3 +36,33 @@ describe('Weibo Data: domain to uid', () => { expect(resData).toBe('1197161814'); }); }); + +describe('statusToHTML', () => { + test('add video poster tag and hide weibo video text', () => { + const videoUrl = 'https://video.weibo.com/show?fid=1034:1234567890&raw=//video'; + const coverUrl = 'https://wx1.sinaimg.cn/orj480/cover.jpg?raw=//cover'; + const shortUrl = 'http://t.cn/A6K3ITkN?foo=1&bar=//raw'; + const videoPosterPattern = /]*?poster\s*=\s*(['"])(.*?)\1[^>]*?>/i; + const html = statusToHTML({ + text: '正文搜狐新闻的微博视频', + page_info: { + type: 'video', + page_url: videoUrl, + url_ori: shortUrl, + page_pic: { + url: coverUrl, + }, + }, + } as WeiboStatus); + + expect(html).not.toContain('搜狐新闻的微博视频'); + expect(html).toContain(''); + expect(html.match(videoPosterPattern)?.[2]).toBe(coverUrl); + expect(html).not.toContain(shortUrl); + expect(html).not.toContain('视频链接'); + expect(html).not.toContain('视频封面'); + expect(html).not.toContain(''); + expect(html).not.toContain(encodeURIComponent(videoUrl)); + expect(html).not.toContain(encodeURIComponent(coverUrl)); + }); +}); diff --git a/src/modules/weibo/weibo.ts b/src/modules/weibo/weibo.ts index a9fd3e7..315a179 100644 --- a/src/modules/weibo/weibo.ts +++ b/src/modules/weibo/weibo.ts @@ -5,10 +5,12 @@ import { createDetailAPI, GetWeiboDetailFunc } from "./api/detailAPI"; import { createDomainAPI, DomainNotFoundError, GetUIDByDomainFunc } from "./api/domainAPI"; import { createIndexAPI, GetIndexUserInfoFunc, GetWeiboContentListFunc, UserNotFoundError } from "./api/indexAPI"; import { createLongTextAPI, GetWeiboLongTextFunc } from "./api/longTextAPI"; +import { WeiboCookieInvalidError } from "./api/common"; export { DomainNotFoundError, UserNotFoundError, + WeiboCookieInvalidError, }; export class WeiboData { @@ -102,12 +104,32 @@ export class WeiboData { fetchUIDByDomain = async (domain: string) => this.getUIDByDomain(domain); } +const firstString = (...values: any[]): string | undefined => values.find(value => typeof value === 'string' && value.length > 0); + +const getVideoUrl = (pageInfo: WeiboStatus['page_info']) => firstString( + pageInfo?.page_url, + pageInfo?.media_info?.h5_url, + pageInfo?.media_info?.mp4_hd_url, + pageInfo?.media_info?.mp4_sd_url, + pageInfo?.media_info?.stream_url_hd, + pageInfo?.media_info?.stream_url, +); + +const getVideoCoverUrl = (pageInfo: WeiboStatus['page_info']) => firstString( + pageInfo?.page_pic?.url, + pageInfo?.media_info?.cover_image_phone, + pageInfo?.media_info?.cover_image, + pageInfo?.media_info?.poster, +); + export const statusToHTML = (status: WeiboStatus) => { let tempHTML = status.text; // 表情转文字 tempHTML = tempHTML.replace(/?(.*?)<\/span>/g, '$1'); // 去掉外链图标 tempHTML = tempHTML.replace(/<\/span>/g, ''); + // 去掉微博视频短链文字 + tempHTML = tempHTML.replace(/[^<]*?的微博视频<\/span>/g, ''); // 转发的微博 if (status.retweeted_status) { @@ -125,11 +147,21 @@ export const statusToHTML = (status: WeiboStatus) => { if (status.pics) { status.pics.forEach(function (item) { tempHTML += "

"; - const url = config.imageCache ? (config.imageCache + encodeURIComponent(item.large.url)) : item.large.url; - const largeUrl = config.imageCache ? (config.imageCache + encodeURIComponent(item.large.url)) : item.large.url; + const url = item.large.url; + const largeUrl = item.large.url; tempHTML += ''; }); } + // 微博视频 + if (status.page_info?.type === 'video') { + const videoUrl = getVideoUrl(status.page_info); + const videoCoverUrl = getVideoCoverUrl(status.page_info); + if (videoCoverUrl) { + const videoSrc = videoUrl ? ' src="' + videoUrl + '"' : ''; + tempHTML += ''; + } + } + return tempHTML; } diff --git a/src/types.ts b/src/types.ts index da39012..7997438 100644 --- a/src/types.ts +++ b/src/types.ts @@ -56,8 +56,22 @@ export interface WeiboStatus { // video and other data page_info?: { type: 'video' | 'search_topic'; + // eg: 'https://video.weibo.com/show?fid=...' + page_url?: string; // eg: 'http://t.cn/A6K3ITkN' url_ori: string; + page_pic?: { + url: string; + [x: string]: any; + }; + media_info?: { + h5_url?: string; + mp4_hd_url?: string; + mp4_sd_url?: string; + stream_url?: string; + stream_url_hd?: string; + [x: string]: any; + }; // eg: '搜狐新闻的微博视频' page_title: string; // eg: '2022触动瞬间'