0.3.0原版
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
node_modules
|
||||
data/rss-data
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
# Created by .ignore support plugin (hsz.mobi)
|
||||
### JetBrains template
|
||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
|
||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||
|
||||
# User-specific stuff:
|
||||
# Sensitive or high-churn files:
|
||||
.idea/**/dataSources/
|
||||
.idea/**/dataSources.ids
|
||||
.idea/**/dataSources.xml
|
||||
.idea/**/dataSources.local.xml
|
||||
.idea/**/sqlDataSources.xml
|
||||
.idea/**/dynamic.xml
|
||||
.idea/**/uiDesigner.xml
|
||||
|
||||
# Gradle:
|
||||
.idea/**/gradle.xml
|
||||
.idea/**/libraries
|
||||
|
||||
# Mongo Explorer plugin:
|
||||
.idea/**/mongoSettings.xml
|
||||
|
||||
## File-based project format:
|
||||
*.iws
|
||||
|
||||
## Plugin-specific files:
|
||||
|
||||
# IntelliJ
|
||||
/out/
|
||||
|
||||
# mpeltonen/sbt-idea plugin
|
||||
.idea_modules/
|
||||
|
||||
.idea/
|
||||
|
||||
# JIRA plugin
|
||||
atlassian-ide-plugin.xml
|
||||
|
||||
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||
com_crashlytics_export_strings.xml
|
||||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
fabric.properties
|
||||
|
||||
/node_modules/
|
||||
|
||||
/data/*
|
||||
!/data/.gitkeep
|
||||
/dist/
|
||||
yarn-error.log
|
||||
|
||||
.DS_Store
|
||||
/config.js
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
FROM node:18-alpine as builder
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
COPY . /app
|
||||
RUN npm i -g pnpm && pnpm i && pnpm build
|
||||
|
||||
FROM node:18-alpine
|
||||
LABEL maintainer="https://github.com/zgq354/weibo-rss"
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
|
||||
# container init
|
||||
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.1/dumb-init_1.2.1_amd64 && \
|
||||
echo "057ecd4ac1d3c3be31f82fc0848bf77b1326a975b4f8423fe31607205a0fe945 /usr/local/bin/dumb-init" | sha256sum -c - && \
|
||||
chmod 755 /usr/local/bin/dumb-init
|
||||
|
||||
COPY package.json pnpm-lock.yaml /app/
|
||||
RUN npm install -g pnpm && pnpm install --prod
|
||||
|
||||
# app code
|
||||
COPY . /app
|
||||
COPY --from=builder /app/dist /app/dist
|
||||
|
||||
EXPOSE 3000
|
||||
ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
|
||||
CMD ["node", "/app/dist/app.js"]
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2017 Zou Guoqing
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,48 @@
|
||||
# weibo-rss
|
||||
|
||||
简单的微博 RSS 订阅源生成器，可将某人最近发布的微博转换为符合 RSS Feed 标准的格式，供阅读器订阅。
|
||||
|
||||
让你不再错过喜欢的博主的动态更新，即使身处纷繁复杂中。
|
||||
|
||||
[](https://heroku.com/deploy)
|
||||
|
||||
## 特点
|
||||
1. 简单：主页链接一键转换 RSS 订阅源地址
|
||||
2. 克制：严格限制程序对微博的并发请求，不产生额外压力
|
||||
3. 省资源：基于 Node.js 实现，采用 [LevelDB](https://github.com/google/leveldb) 在本地文件系统做 cache，内存占用低（60MB 左右）
|
||||
|
||||
## 手动部署
|
||||
|
||||
依赖：`Node.js` 和 `pnpm`
|
||||
|
||||
安装：
|
||||
```
|
||||
git clone https://github.com/zgq354/weibo-rss.git
|
||||
cd weibo-rss
|
||||
pnpm i && pnpm build
|
||||
```
|
||||
|
||||
启动：
|
||||
```
|
||||
pnpm install pm2 -g
|
||||
pm2 start process.json
|
||||
```
|
||||
|
||||
程序将启动一个 HTTP Server，默认监听 `3000` 端口
|
||||
还需另外配置域名、HTTP 反向代理等
|
||||
|
||||
## ToDo
|
||||
1. 更多的接口单元测试
|
||||
|
||||
## 贡献者们
|
||||
<a href="https://github.com/zgq354/weibo-rss/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=zgq354/weibo-rss" />
|
||||
</a>
|
||||
|
||||
## 相关项目
|
||||
|
||||
* [RSSHub](https://github.com/DIYgod/RSSHub)
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "weibo-rss",
|
||||
"description": "Heroku Weibo Rss Generator",
|
||||
"repository": "https://github.com/zgq354/weibo-rss",
|
||||
"logo": "https://huan.github.io/weibo-rss/images/weibo-rss.png",
|
||||
"keywords": [
|
||||
"weibo",
|
||||
"rss"
|
||||
],
|
||||
"website": "https://zgq.ink/posts/weibo-rss",
|
||||
"success_url": "/",
|
||||
"env": {
|
||||
"EXAMPLE": {
|
||||
"description": "Example",
|
||||
"required": false
|
||||
}
|
||||
},
|
||||
"engines": {
|
||||
"node": "10"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
weibo-rss:
|
||||
build: .
|
||||
container_name: docker-weibo-rss
|
||||
restart: always
|
||||
ports:
|
||||
- "3000:3000"
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 7.8 KiB |
@@ -0,0 +1,5 @@
|
||||
/** @type {import('ts-jest').JestConfigWithTsJest} */
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
};
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"name": "weibo-rss",
|
||||
"version": "0.3.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "ts-node src/app.ts",
|
||||
"serve": "node dist/app.js",
|
||||
"dev": "cross-env DEBUG=1 ts-node-dev src/app.ts",
|
||||
"test": "jest"
|
||||
},
|
||||
"dependencies": {
|
||||
"@koa/router": "^12.0.0",
|
||||
"axios": "^1.16.1",
|
||||
"koa": "^2.16.4",
|
||||
"koa-static": "^5.0.0",
|
||||
"level": "^6.0.1",
|
||||
"leveldown": "^6.1.1",
|
||||
"levelup": "^5.1.1",
|
||||
"node-schedule": "^1.3.0",
|
||||
"np-queue": "^2.0.0",
|
||||
"rss": "^1.2.2",
|
||||
"tracer": "^1.1.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@jest/globals": "^29.7.0",
|
||||
"@types/koa": "^2.15.2",
|
||||
"@types/koa-static": "^4.0.2",
|
||||
"@types/koa__router": "^12.0.5",
|
||||
"@types/leveldown": "^4.0.2",
|
||||
"@types/levelup": "^4.3.0",
|
||||
"@types/node": "^14.18.36",
|
||||
"@types/node-schedule": "^1.3.1",
|
||||
"@types/rss": "^0.0.29",
|
||||
"cross-env": "^7.0.3",
|
||||
"jest": "^29.7.0",
|
||||
"ts-jest": "^29.4.11",
|
||||
"ts-node": "^10.9.1",
|
||||
"ts-node-dev": "^2.0.0",
|
||||
"typescript": "^4.9.4"
|
||||
}
|
||||
}
|
||||
Generated
+4275
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
allowBuilds:
|
||||
leveldown: true
|
||||
|
||||
overrides:
|
||||
'@babel/helpers': ^7.26.10
|
||||
brace-expansion: ^1.1.13
|
||||
cross-spawn: ^7.0.6
|
||||
diff: ^4.0.4
|
||||
js-yaml: ^3.14.2
|
||||
minimatch: ^3.1.5
|
||||
picomatch: ^2.3.2
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"apps": [
|
||||
{
|
||||
"name": "weibo-rss",
|
||||
"script": "node ./dist/app.js",
|
||||
"watch": true,
|
||||
"env": {
|
||||
"PORT": "3000"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+6
File diff suppressed because one or more lines are too long
Binary file not shown.
|
After Width: | Height: | Size: 66 KiB |
@@ -0,0 +1,135 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="renderer" content="webkit" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
|
||||
<title>微博RSS订阅生成器</title>
|
||||
<link rel="stylesheet" href="css/bootstrap.min.css">
|
||||
<link rel="shortcut icon" href="favicon.ico"/>
|
||||
<style>
|
||||
.content {
|
||||
max-width: 500px;
|
||||
margin-left: auto;
|
||||
margin-right: auto;
|
||||
padding: 15px;
|
||||
}
|
||||
.head {
|
||||
margin-top: 70px;
|
||||
}
|
||||
.intro {
|
||||
margin-top: 30px;
|
||||
}
|
||||
.loading {
|
||||
margin-top: 20px;
|
||||
}
|
||||
.result {
|
||||
margin-top: 20px;
|
||||
}
|
||||
.footer-bottom {
|
||||
margin-top: 100px;
|
||||
}
|
||||
.reader-list {
|
||||
margin-top: 20px;
|
||||
}
|
||||
#form {
|
||||
margin-top: 18px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="row">
|
||||
<div class="content">
|
||||
<div class="head">
|
||||
<h2 class="text-center">Weibo to RSS</h2>
|
||||
<p class="intro">优雅地使用 RSS 阅读器订阅喜欢的微博博主</p>
|
||||
<p>食用方式：<br>1. 在输入框中粘贴你想订阅的微博地址<br>2. 点击“生成”按钮，获得RSS订阅源</p>
|
||||
<form id="form">
|
||||
<div class="input-group">
|
||||
<input id="text-input" type="text" class="form-control" placeholder="https://weibo.com/kaifulee">
|
||||
<span class="input-group-btn">
|
||||
<button class="btn btn-default">生成</button>
|
||||
</span>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="loading" style="display: none;"><p class="text-center">加载中...</p></div>
|
||||
<div class="result" style="display: none;">
|
||||
<p>生成的RSS订阅源：</p>
|
||||
<div class="input-group">
|
||||
<input id="foo" type="text" class="form-control" value="">
|
||||
<span class="input-group-btn">
|
||||
<button id="copy-btn" data-clipboard-action="copy" data-clipboard-target="#foo" class="btn btn-default">复制到剪贴板</button>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="footer-bottom">
|
||||
<div class="container">
|
||||
<p class="text-mute text-center">Powered by <a href="https://github.com/zgq354/weibo-rss" target="_blank">weibo-rss</a>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
<script src="js/jquery-3.2.1.min.js"></script>
|
||||
<script src="js/clipboard.min.js"></script>
|
||||
<script>
|
||||
var prefix = {
|
||||
inoreader: "https://www.inoreader.com/?add_feed=",
|
||||
feedly: "https://cloud.feedly.com/#subscription/feed/",
|
||||
};
|
||||
|
||||
// 初始化Clipboard插件
|
||||
var clipboard = new Clipboard('#copy-btn');
|
||||
clipboard.on('success', function(e) {
|
||||
console.log(e);
|
||||
});
|
||||
clipboard.on('error', function(e) {
|
||||
console.log(e);
|
||||
});
|
||||
|
||||
// 链接处理
|
||||
$('#form').on('submit', function (e) {
|
||||
e.preventDefault();
|
||||
$(".result").hide();
|
||||
$(".loading").show();
|
||||
|
||||
var input = $("#text-input").val();
|
||||
var temp = input.match(/weibo\.(com|cn)\/(u|(profile))?\/?(\d{10})/);
|
||||
if (!temp) {
|
||||
temp = input.match(/weibo\.(com|cn)\/(u|(profile))?\/?([^?]+?)($|\?)/);
|
||||
if (!temp) {
|
||||
showResult(false);
|
||||
} else {
|
||||
$.get("api/domain2uid?domain=" + temp[4], function (data) {
|
||||
showResult(data.uid);
|
||||
}).fail(function (err) {
|
||||
showResult(false);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
showResult(temp[4]);
|
||||
}
|
||||
});
|
||||
// 显示结果
|
||||
function showResult(uid) {
|
||||
$(".loading").hide();
|
||||
// console.log(uid);
|
||||
if (!uid) {
|
||||
alert("数据获取失败");
|
||||
return;
|
||||
}
|
||||
var baseUrl = location.href.split("?")[0];
|
||||
if (baseUrl.substr(-1, 1) !== "/") baseUrl += "/";
|
||||
var feedUrl = baseUrl + "rss/user/" + uid;
|
||||
|
||||
$("#foo").val(feedUrl);
|
||||
for (var reader in prefix) {
|
||||
$("#" + reader).attr("href", prefix[reader] + feedUrl);
|
||||
}
|
||||
$(".result").show();
|
||||
}
|
||||
</script>
|
||||
</html>
|
||||
Vendored
+7
File diff suppressed because one or more lines are too long
Vendored
+4
File diff suppressed because one or more lines are too long
+53
@@ -0,0 +1,53 @@
|
||||
import Koa from 'koa';
|
||||
import Router from '@koa/router';
|
||||
import serve from 'koa-static';
|
||||
import path from 'path';
|
||||
import config from './config';
|
||||
import { logger } from "./modules/logger";
|
||||
import { normalizePort } from './utils';
|
||||
import { registerRoutes } from './modules/routes';
|
||||
import { RSSKoaContext, RSSKoaState } from './types';
|
||||
import { LevelCache } from './modules/cache';
|
||||
import { WeiboData } from './modules/weibo/weibo';
|
||||
|
||||
const koaApp = new Koa<RSSKoaState, RSSKoaContext>();
|
||||
const initApp = () => {
|
||||
const router = new Router<RSSKoaState, RSSKoaContext>();
|
||||
registerRoutes(router);
|
||||
|
||||
// levelCache
|
||||
const cache = LevelCache.getInstance(path.join(config.rootDir, 'data'), logger);
|
||||
cache.startScheduleCleanJob();
|
||||
|
||||
// weibo
|
||||
const weiboData = new WeiboData(cache, logger);
|
||||
|
||||
// enable X-Forwarded-For
|
||||
koaApp.proxy = true;
|
||||
|
||||
// init middleware
|
||||
return koaApp
|
||||
.use(async (ctx, next) => {
|
||||
// duration log
|
||||
const startTime = Date.now();
|
||||
logger.debug(`${ctx.req.method} ${ctx.originalUrl} ${ctx.ip}`);
|
||||
await next();
|
||||
const duration = Date.now() - startTime;
|
||||
logger.info(`[${ctx.status}] ${ctx.req.method} ${ctx.originalUrl} ${ctx.ip} hit: ${ctx.state.hit || 0} ${duration}ms`);
|
||||
})
|
||||
.use(async (ctx, next) => {
|
||||
ctx.cache = cache;
|
||||
ctx.weibo = weiboData;
|
||||
await next();
|
||||
})
|
||||
.use(serve(config.rootDir + '/public'))
|
||||
.use(router.routes());
|
||||
};
|
||||
|
||||
// start service
|
||||
initApp();
|
||||
const port = normalizePort(process.env.PORT || config.port);
|
||||
koaApp.listen(port, () => {
|
||||
logger.info(`weibo-rss start`);
|
||||
logger.info(`Listening http://0.0.0.0:${port}`);
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* 配置文件加载相关
|
||||
*/
|
||||
import { existsSync } from "fs";
|
||||
import { logger } from "./modules/logger";
|
||||
|
||||
const rootDir = `${__dirname}/../`;
|
||||
let customConfig = {};
|
||||
|
||||
// 加载自定义配置
|
||||
if (existsSync(`${rootDir}/config.js`)) {
|
||||
try {
|
||||
customConfig = require('../config');
|
||||
} catch (err) {
|
||||
logger.error('custom config load failed,', err);
|
||||
}
|
||||
}
|
||||
|
||||
// 默认配置
|
||||
const defaultConfig = {
|
||||
// 程序监听的 TCP 端口，也可以在环境变量指定
|
||||
port: '3000',
|
||||
// 输出 rss feed 文本的 ttl 字段（分钟为单位）
|
||||
rssTTL: 15,
|
||||
// 真正的缓存配置，具体关注代码（秒为单位）
|
||||
cacheTTL: {
|
||||
rssXml: 15 * 60,
|
||||
apiStatusList: 15 * 60,
|
||||
apiIndexInfo: 1 * 24 * 60 * 60,
|
||||
apiLongText: 7 * 24 * 60 * 60,
|
||||
apiDetail: 7 * 24 * 60 * 60,
|
||||
apiDomain: 7 * 24 * 60 * 60,
|
||||
},
|
||||
// 图片缓存代理
|
||||
imageCache: 'https://image.baidu.com/search/down?url=',
|
||||
};
|
||||
|
||||
/**
|
||||
* 自定义配置
|
||||
*/
|
||||
export default {
|
||||
...defaultConfig,
|
||||
...customConfig,
|
||||
rootDir, // 站点根路径
|
||||
};
|
||||
@@ -0,0 +1,158 @@
|
||||
import LevelDOWN from "leveldown";
|
||||
import levelUp, { LevelUp } from "levelup";
|
||||
import { scheduleJob } from "node-schedule";
|
||||
import path from "path";
|
||||
import { CacheInterface, LoggerInterface } from "../types";
|
||||
import { logger } from "./logger";
|
||||
|
||||
// 缓存条目的结构
|
||||
export interface CacheObject {
|
||||
created: number; // 缓存设置时的时间戳（毫秒）
|
||||
expire: boolean | number; // 有效期（秒），falsy 为不清理
|
||||
value: any;
|
||||
};
|
||||
|
||||
// levelup 的错误类型
|
||||
// https://github.com/Level/errors/blob/f5e5e406a5e325a8a62eb4d1135fa3d010816f8b/errors.js
|
||||
export interface NotFoundError extends Error {
|
||||
notFound: boolean;
|
||||
}
|
||||
|
||||
// 默认缓存目录
|
||||
const DB_FOLDER = 'rss-data';
|
||||
|
||||
/**
|
||||
* LevelDB 实现的文件系统缓存
|
||||
*/
|
||||
export class LevelCache implements CacheInterface {
|
||||
private instance: LevelUp;
|
||||
private logger: LoggerInterface;
|
||||
|
||||
constructor(dbPath: string, logger: LoggerInterface) {
|
||||
this.instance = levelUp(LevelDOWN(dbPath));
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
static instance: LevelCache = null;
|
||||
|
||||
static getInstance(dataBaseDir: string, log: LoggerInterface = logger) {
|
||||
if (!LevelCache.instance) {
|
||||
LevelCache.instance = new LevelCache(path.join(dataBaseDir, DB_FOLDER), log);
|
||||
}
|
||||
return LevelCache.instance;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置缓存
|
||||
* @param key 需缓存的 key
|
||||
* @param value key 对应的值
|
||||
* @param expire 过期时间（单位秒）
|
||||
*/
|
||||
set(key: string, value: any, expire: number = 0) {
|
||||
this.logger.debug(`[cache] set ${key}`);
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const data: CacheObject = {
|
||||
created: Date.now(),
|
||||
expire: !!expire,
|
||||
value,
|
||||
};
|
||||
if (expire) {
|
||||
data.expire = expire;
|
||||
}
|
||||
this.instance.put(key, JSON.stringify(data), (err) => {
|
||||
if (err) return reject(err);
|
||||
return resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取缓存的值
|
||||
* @param key 缓存对象的 key
|
||||
*/
|
||||
get(key: string) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.instance.get(key, (err: NotFoundError, value) => {
|
||||
if (err) {
|
||||
if (err.notFound) {
|
||||
this.logger.debug(`[cache] get ${key} notFound`);
|
||||
return resolve(null);
|
||||
}
|
||||
this.logger.error(`[cache] get ${key} error`, err);
|
||||
return reject(err);
|
||||
}
|
||||
try {
|
||||
const data = JSON.parse(String(value)) as CacheObject;
|
||||
if (this.checkExpired(data)) {
|
||||
this.logger.debug(`[cache] get ${key} expired`);
|
||||
// 过期缓存交给定时任务清理
|
||||
return resolve(null);
|
||||
} else {
|
||||
this.logger.debug(`[cache] get ${key}`);
|
||||
return resolve(data.value);
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`[cache] parse ${key} error`, error);
|
||||
return reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* memo in cache
|
||||
*/
|
||||
async memo<T>(cb: () => T, key: string, expire = 0): Promise<Awaited<T>> {
|
||||
const cacheResp = await this.get(key);
|
||||
if (cacheResp) {
|
||||
return cacheResp as Awaited<T>;
|
||||
}
|
||||
const res = await cb();
|
||||
this.set(key, res, expire);
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* 定期清理过期缓存
|
||||
*/
|
||||
startScheduleCleanJob(rule: Parameters<typeof scheduleJob>[0] = "0 30 2 * * *") {
|
||||
// 每日凌晨两点半开始处理
|
||||
return scheduleJob(rule, () => {
|
||||
this.logger.info("[cache] cleaning start");
|
||||
let total = 0,
|
||||
deleted = 0;
|
||||
this.instance
|
||||
.createReadStream()
|
||||
.on("data", (item) => {
|
||||
total++;
|
||||
try {
|
||||
const data = JSON.parse(item.value.toString()) as CacheObject;
|
||||
if (this.checkExpired(data)) {
|
||||
deleted++;
|
||||
setTimeout(() => {
|
||||
this.instance.del(item.key, (err) => {
|
||||
if (err) {
|
||||
this.logger.error(`[cache] delete err key: ${item.key}`, err);
|
||||
}
|
||||
});
|
||||
}, 0);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error("[cache] parse error", err);
|
||||
}
|
||||
})
|
||||
.on("end", () => {
|
||||
this.logger.info(
|
||||
"[cache] cleaning finished, total cnt:",
|
||||
total,
|
||||
"deleted cnt:",
|
||||
deleted
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private checkExpired(cacheItem: CacheObject) {
|
||||
return cacheItem.expire && Date.now() - cacheItem.created > +cacheItem.expire * 1000;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* 日志模块，生成对应格式日志输出到控制台
|
||||
*/
|
||||
import * as tracer from 'tracer';
|
||||
|
||||
const debug = process.env.DEBUG;
|
||||
|
||||
export const logger = tracer.colorConsole({
|
||||
format : "[{{timestamp}}] [{{title}}] {{message}}",
|
||||
level: debug ? 'debug' : 'info',
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
/**
|
||||
* URL 路由分发
|
||||
*/
|
||||
import Router from '@koa/router';
|
||||
import NodeRSS from 'rss';
|
||||
import { RSSKoaContext, RSSKoaState } from '../types';
|
||||
import config from '../config';
|
||||
import { DomainNotFoundError, statusToHTML, UserNotFoundError } from './weibo/weibo';
|
||||
import { ThrottledError } from './throttler';
|
||||
import { logger } from './logger';
|
||||
|
||||
export class UidInvalidError extends Error {
|
||||
constructor(uid: string) {
|
||||
super(`uid: ${uid}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class DomainInvalidError extends Error {
|
||||
constructor(domain: string) {
|
||||
super(`domain: ${domain}`);
|
||||
}
|
||||
}
|
||||
|
||||
export const registerRoutes = (
|
||||
router: Router<RSSKoaState, RSSKoaContext>
|
||||
) => {
|
||||
router.get('/rss/user/:id', async (ctx) => {
|
||||
const uid = ctx.params['id'];
|
||||
try {
|
||||
// check uid format
|
||||
if (!/^[0-9]{10}$/.test(uid)) {
|
||||
throw new UidInvalidError(uid);
|
||||
}
|
||||
|
||||
// get data
|
||||
let cacheMiss = false;
|
||||
const xmlData = await ctx.cache.memo(async () => {
|
||||
const weiboData = await ctx.weibo.fetchUserLatestWeibo(uid);
|
||||
if (weiboData) {
|
||||
// basic info
|
||||
const feed = new NodeRSS({
|
||||
site_url: "https://weibo.com/" + uid,
|
||||
feed_url: '',
|
||||
title: weiboData.screenName + '的微博',
|
||||
description: weiboData.description,
|
||||
generator: 'https://github.com/zgq354/weibo-rss',
|
||||
ttl: config.rssTTL,
|
||||
});
|
||||
// item
|
||||
weiboData.statusList?.forEach((status) => {
|
||||
if (!status) return;
|
||||
feed.item({
|
||||
title: status.status_title || (status.text ? status.text.replace(/<[^>]+>/g, '').replace(/[\n]/g, '').substr(0, 25) : null),
|
||||
description: statusToHTML(status),
|
||||
url: 'https://weibo.com/' + uid + '/' + status.bid,
|
||||
date: new Date(status.created_at),
|
||||
});
|
||||
});
|
||||
cacheMiss = true;
|
||||
return feed.xml();
|
||||
}
|
||||
}, `xml-${uid}`, config.cacheTTL.rssXml);
|
||||
|
||||
// send data
|
||||
ctx.set('Content-Type', 'text/xml');
|
||||
ctx.body = xmlData;
|
||||
|
||||
// mark hit cache
|
||||
ctx.state.hit = cacheMiss ? 0 : 1;
|
||||
} catch (error) {
|
||||
if (error instanceof UidInvalidError) {
|
||||
ctx.status = 404;
|
||||
ctx.body = `找不到用户，传入 UID 格式有误。uid: ${uid}`;
|
||||
return;
|
||||
}
|
||||
if (error instanceof UserNotFoundError) {
|
||||
ctx.status = 404;
|
||||
ctx.body = `找不到用户，可能用户仅登录可见，不支持订阅。可以通过打开 https://m.weibo.cn/u/:uid 验证（<a href="https://m.weibo.cn/u/${uid}" target="_blank">uid: ${uid}</a>）`;
|
||||
return;
|
||||
}
|
||||
if (error instanceof ThrottledError) {
|
||||
ctx.status = 503;
|
||||
ctx.body = `暂时无法拉取到数据，请稍后再试。uid: ${uid}`;
|
||||
return;
|
||||
}
|
||||
ctx.status = 500;
|
||||
ctx.body = `未知错误，需管理员检查日志。uid: ${uid}`;
|
||||
logger.error(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/api/domain2uid', async (ctx) => {
|
||||
const domain = ctx.request.query['domain'] as string;
|
||||
try {
|
||||
// verify
|
||||
if (!domain || !/^[A-Za-z0-9]{3,20}$/.test(domain)) {
|
||||
throw new DomainInvalidError(domain);
|
||||
}
|
||||
// start fetching
|
||||
let cacheMiss = false;
|
||||
const uid = await ctx.cache.memo(
|
||||
() => {
|
||||
cacheMiss = true;
|
||||
return ctx.weibo.fetchUIDByDomain(domain);
|
||||
},
|
||||
`dm-${domain}`,
|
||||
config.cacheTTL.apiDomain,
|
||||
);
|
||||
logger.debug(`domain: ${domain}, uid: ${uid}`);
|
||||
ctx.body = {
|
||||
success: true,
|
||||
uid
|
||||
};
|
||||
|
||||
// mark hit cache
|
||||
ctx.state.hit = cacheMiss ? 0 : 1;
|
||||
} catch (error) {
|
||||
if (error instanceof DomainInvalidError || error instanceof DomainNotFoundError) {
|
||||
ctx.status = 404;
|
||||
ctx.body = {
|
||||
success: false,
|
||||
msg: '找不到用户，可能是地址格式不正确',
|
||||
};
|
||||
return;
|
||||
}
|
||||
logger.error(error);
|
||||
ctx.status = 500;
|
||||
ctx.body = {
|
||||
success: false,
|
||||
msg: '获取数据时发生了错误'
|
||||
};
|
||||
}
|
||||
})
|
||||
};
|
||||
@@ -0,0 +1,52 @@
|
||||
import Queue from "np-queue";
|
||||
import { LoggerInterface } from "../types";
|
||||
import { logger } from "./logger";
|
||||
|
||||
export class ThrottledError extends Error {
|
||||
constructor() {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Limit the request frequency
|
||||
*/
|
||||
export class Throttler {
|
||||
name: string = '';
|
||||
queue: Queue;
|
||||
enable: boolean;
|
||||
lastUpdateTime: number;
|
||||
retryDelayMs: number;
|
||||
logger: LoggerInterface;
|
||||
|
||||
constructor(name = '', log: LoggerInterface = logger) {
|
||||
this.name = name;
|
||||
this.logger = log;
|
||||
this.queue = new Queue({ concurrency: 1 });
|
||||
this.retryDelayMs = 600000;
|
||||
this.enable = true;
|
||||
this.lastUpdateTime = Date.now();
|
||||
}
|
||||
|
||||
checkFuncAvailable = () => {
|
||||
if (!this.enable && Date.now() - this.lastUpdateTime > this.retryDelayMs) {
|
||||
this.enable = true;
|
||||
this.logger.info(`[Throttled] reenabled ${this.name}`);
|
||||
}
|
||||
return this.enable;
|
||||
};
|
||||
|
||||
disableFunc = () => {
|
||||
this.enable = false;
|
||||
this.lastUpdateTime = Date.now();
|
||||
this.logger.info(`[Throttled] disabled ${this.name}`);
|
||||
return Promise.reject(new ThrottledError());
|
||||
};
|
||||
|
||||
runFunc = <T = any>(callback: (disable: () => Promise<void>) => any): Promise<T> => {
|
||||
if (!this.checkFuncAvailable()) {
|
||||
return Promise.reject(new ThrottledError());
|
||||
}
|
||||
return this.queue.add(async () => callback(this.disableFunc));
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { AxiosError } from "axios";
|
||||
|
||||
export const TIME_OUT = 3000;
|
||||
export const MOCK_UA = 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36';
|
||||
|
||||
export const handleForbiddenErr = (err: AxiosError, cb: () => Promise<void>) => {
|
||||
if (err.response && [418, 403].includes(err.response.status)) {
|
||||
return cb();
|
||||
} else {
|
||||
return Promise.reject(err);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
// TODO: add more tests
|
||||
import { describe, expect, test } from "@jest/globals";
|
||||
|
||||
describe('Weibo API: detailAPI', () => {
|
||||
test('TODO: add more tests', async () => {
|
||||
expect(1).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
import { Throttler } from "../../throttler";
|
||||
import Axios from "axios";
|
||||
import { Agent } from "https";
|
||||
import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
|
||||
import { logger } from "../../logger";
|
||||
import { waitMs } from "../../../utils";
|
||||
|
||||
export const createDetailAPI = () => {
|
||||
const runner = new Throttler('detail');
|
||||
const httpsAgent = new Agent({ keepAlive: true });
|
||||
const axiosInstance = Axios.create({
|
||||
timeout: TIME_OUT,
|
||||
httpsAgent
|
||||
});
|
||||
return {
|
||||
getWeiboDetail: (id: string) => runner.runFunc(async (disable) => {
|
||||
logger.debug(`[getDetail] ${id}`);
|
||||
await waitMs(Math.floor(Math.random() * 100));
|
||||
return await axiosInstance({
|
||||
method: 'get',
|
||||
url: `https://m.weibo.cn/statuses/show?id=${id}`,
|
||||
headers: {
|
||||
'Mweibo-Pwa': 1,
|
||||
'Referer': `https://m.weibo.cn/detail/${id}`,
|
||||
'User-Agent': MOCK_UA,
|
||||
'X-Requested-With': 'XMLHttpRequest'
|
||||
},
|
||||
}).then(({ data }) => {
|
||||
return data.data;
|
||||
}).catch(err => handleForbiddenErr(err, disable));
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export type GetWeiboDetailFunc = ReturnType<typeof createDetailAPI>['getWeiboDetail'];
|
||||
@@ -0,0 +1,27 @@
|
||||
import { describe, expect, test } from "@jest/globals";
|
||||
import { createDomainAPI } from "./domainAPI";
|
||||
|
||||
const testDomainMapList = [
|
||||
{
|
||||
domain: 'kaifulee',
|
||||
uid: '1197161814',
|
||||
},
|
||||
{
|
||||
domain: 'taobao',
|
||||
uid: '1682454721',
|
||||
},
|
||||
{
|
||||
domain: 'tmall',
|
||||
uid: '1768198384',
|
||||
},
|
||||
];
|
||||
|
||||
describe('Domain API: domain to uid', () => {
|
||||
test('basic domain format', async () => {
|
||||
const { getUIDByDomain } = createDomainAPI();
|
||||
testDomainMapList.forEach(async (data) => {
|
||||
const resData = await getUIDByDomain(data.domain);
|
||||
expect(resData).toBe(data.uid);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
import { Throttler } from "../../throttler";
|
||||
import Axios from "axios";
|
||||
import { Agent } from "https";
|
||||
import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
|
||||
import { logger } from "../../logger";
|
||||
import { waitMs } from "../../../utils";
|
||||
|
||||
export class DomainNotFoundError extends Error {
|
||||
constructor(domain: string) {
|
||||
super(`domain: ${domain}`);
|
||||
}
|
||||
}
|
||||
|
||||
export const createDomainAPI = () => {
|
||||
const runner = new Throttler('domain');
|
||||
const httpsAgent = new Agent({ keepAlive: true });
|
||||
const axiosInstance = Axios.create({
|
||||
timeout: TIME_OUT,
|
||||
httpsAgent
|
||||
});
|
||||
|
||||
return {
|
||||
getUIDByDomain: (domain: string) => runner.runFunc<string>(async (disable) => {
|
||||
logger.debug(`[domain] convert ${domain}`);
|
||||
await waitMs(Math.floor(Math.random() * 100));
|
||||
return await axiosInstance.get(`https://m.weibo.cn/${domain}?&jumpfrom=weibocom`, {
|
||||
headers: {
|
||||
'User-Agent': MOCK_UA,
|
||||
},
|
||||
}).then(res => {
|
||||
const uid = res.request.path.split("/u/")[1] as string;
|
||||
if (!uid) {
|
||||
throw new DomainNotFoundError(domain);
|
||||
}
|
||||
return uid;
|
||||
}).catch(err => handleForbiddenErr(err, disable));
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
export type GetUIDByDomainFunc = ReturnType<typeof createDomainAPI>['getUIDByDomain'];
|
||||
@@ -0,0 +1,8 @@
|
||||
// TODO: add more tests
|
||||
import { describe, expect, test } from "@jest/globals";
|
||||
|
||||
describe('Weibo API: indexAPI', () => {
|
||||
test('TODO: add more tests', async () => {
|
||||
expect(1).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
import { Throttler } from "../../throttler";
|
||||
import Axios from "axios";
|
||||
import { Agent } from "https";
|
||||
import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
|
||||
import { WeiboStatus, WeiboUserData } from "../../../types";
|
||||
import { logger } from "../../logger";
|
||||
import { waitMs } from "../../../utils";
|
||||
|
||||
export class UserNotFoundError extends Error {
|
||||
constructor(uid: string) {
|
||||
super(`uid: ${uid}`);
|
||||
}
|
||||
}
|
||||
|
||||
export const createIndexAPI = () => {
|
||||
const runner = new Throttler('index');
|
||||
const httpsAgent = new Agent({ keepAlive: true });
|
||||
const axiosInstance = Axios.create({
|
||||
timeout: TIME_OUT,
|
||||
httpsAgent
|
||||
});
|
||||
|
||||
return {
|
||||
getIndexUserInfo: (uid: string) => runner.runFunc<WeiboUserData>(async (disable) => {
|
||||
const resHeaders = await axiosInstance({
|
||||
"method": "POST",
|
||||
"url": "https://visitor.passport.weibo.cn/visitor/genvisitor2",
|
||||
"headers": {
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"User-Agent": MOCK_UA,
|
||||
},
|
||||
"data": {
|
||||
"cb": "visitor_gray_callback",
|
||||
tid: '',
|
||||
from: 'weibo'
|
||||
}
|
||||
}).then(res => res.headers)
|
||||
axiosInstance.defaults.headers.common['Cookie'] = resHeaders['set-cookie'].filter(cookie => cookie.startsWith('SUB')).map(cookie => cookie.split(';')[0]).join(';');
|
||||
logger.debug(`[getInfo] ${uid}`);
|
||||
await waitMs(Math.floor(Math.random() * 100));
|
||||
return await axiosInstance({
|
||||
method: 'get',
|
||||
url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
|
||||
headers: {
|
||||
'Mweibo-Pwa': 1,
|
||||
'Referer': `https://m.weibo.cn/u/${uid}`,
|
||||
'User-Agent': MOCK_UA,
|
||||
'X-Requested-With': 'XMLHttpRequest'
|
||||
}
|
||||
}).then(({ data }) => {
|
||||
if (data.ok !== 1) {
|
||||
return Promise.reject(new UserNotFoundError(uid));
|
||||
}
|
||||
const result = {
|
||||
uid,
|
||||
screenName: data.data.userInfo.screen_name,
|
||||
description: data.data.userInfo.description,
|
||||
containerId: data.data.tabsInfo.tabs[1].containerid,
|
||||
};
|
||||
return result;
|
||||
}).catch(err => handleForbiddenErr(err, disable));
|
||||
}),
|
||||
getWeiboContentList: (uid: string, containerId: string) => runner.runFunc<WeiboStatus[]>(async (disable) => {
|
||||
logger.debug(`[getContList] ${uid} ${containerId}`);
|
||||
await waitMs(Math.floor(Math.random() * 100));
|
||||
return await axiosInstance({
|
||||
method: 'get',
|
||||
url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}&containerid=${containerId}`,
|
||||
headers: {
|
||||
'Mweibo-Pwa': 1,
|
||||
'Referer': `https://m.weibo.cn/u/${uid}`,
|
||||
'User-Agent': MOCK_UA,
|
||||
'X-Requested-With': 'XMLHttpRequest'
|
||||
}
|
||||
}).then(({ data }) => {
|
||||
return data.data.cards
|
||||
.filter(item => item.mblog)
|
||||
.map(item => item.mblog);
|
||||
}).catch(err => handleForbiddenErr(err, disable));
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
export type GetIndexUserInfoFunc = ReturnType<typeof createIndexAPI>['getIndexUserInfo'];
|
||||
|
||||
export type GetWeiboContentListFunc = ReturnType<typeof createIndexAPI>['getWeiboContentList'];
|
||||
@@ -0,0 +1,20 @@
|
||||
import { describe, expect, test } from "@jest/globals";
|
||||
import { createLongTextAPI } from "./longTextAPI";
|
||||
|
||||
const testDataList = [{
|
||||
id: '5093426468489917',
|
||||
text: '中微子是宇宙形成之初就存在的最古老也最原始的基本粒子',
|
||||
}, {
|
||||
id: '5093698779485067',
|
||||
text: '无论如何，「AI God」的拍卖再次引发了人们对传统艺术与数字艺术的思考',
|
||||
}];
|
||||
|
||||
describe ('Weibo API: longTextAPI', () => {
|
||||
test('fetch long text', async () => {
|
||||
const { getWeiboLongText } = createLongTextAPI();
|
||||
for (const data of testDataList) {
|
||||
const resData = await getWeiboLongText(data.id);
|
||||
expect(resData).toContain(data.text);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import { Throttler } from "../../throttler";
|
||||
import Axios from "axios";
|
||||
import { Agent } from "https";
|
||||
import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
|
||||
import { logger } from "../../logger";
|
||||
import { waitMs } from "../../../utils";
|
||||
|
||||
export const createLongTextAPI = () => {
|
||||
const runner = new Throttler('longText');
|
||||
const httpsAgent = new Agent({ keepAlive: true });
|
||||
const axiosInstance = Axios.create({
|
||||
timeout: TIME_OUT,
|
||||
httpsAgent
|
||||
});
|
||||
return {
|
||||
getWeiboLongText: (id: string) => runner.runFunc<string>(async (disable) => {
|
||||
logger.debug(`[longText] ${id}`);
|
||||
await waitMs(Math.floor(Math.random() * 100));
|
||||
return await axiosInstance({
|
||||
method: 'get',
|
||||
url: `https://m.weibo.cn/statuses/extend?id=${id}`,
|
||||
headers: {
|
||||
'Mweibo-Pwa': 1,
|
||||
'Referer': `https://m.weibo.cn/detail/${id}`,
|
||||
'User-Agent': MOCK_UA,
|
||||
'X-Requested-With': 'XMLHttpRequest'
|
||||
},
|
||||
}).then(({ data }) => {
|
||||
if (!data.data) {
|
||||
throw new Error(JSON.stringify(data));
|
||||
}
|
||||
return data.data.longTextContent;
|
||||
}).catch(err => handleForbiddenErr(err, disable));
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
export type GetWeiboLongTextFunc = ReturnType<typeof createLongTextAPI>['getWeiboLongText'];
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, test } from '@jest/globals';
|
||||
import { WeiboData } from "./weibo";
|
||||
import { WeiboStatus } from '../../types';
|
||||
|
||||
const wbData = new WeiboData({
|
||||
set: async () => null,
|
||||
get: async () => null,
|
||||
memo: async <T>(cb: () => T): Promise<Awaited<T>> => await cb(),
|
||||
});
|
||||
|
||||
// TODO: more unit test about weibo data
|
||||
describe('Weibo Data: user weibo profile and list', () => {
|
||||
const TEST_UID = '5890672121';
|
||||
|
||||
// 获取用户名 + 微博列表
|
||||
test('fetch user weibo profile and list success', async () => {
|
||||
const resData = await wbData.fetchUserLatestWeibo(TEST_UID);
|
||||
expect(resData.uid).toBe(TEST_UID);
|
||||
expect(resData.screenName).toBe('搜狐新闻');
|
||||
expect(resData.statusList).toBeDefined();
|
||||
});
|
||||
|
||||
// 长文本的填充
|
||||
test('fetch weibo with long text', async () => {
|
||||
const resData = await wbData.fillStatusWithLongText({
|
||||
id: '5093426468489917',
|
||||
isLongText: true,
|
||||
} as WeiboStatus);
|
||||
expect(resData.text).toContain('中微子是宇宙形成之初就存在的最古老也最原始的基本粒子');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Weibo Data: domain to uid', () => {
|
||||
test('basic domain format', async () => {
|
||||
const resData = await wbData.fetchUIDByDomain('kaifulee');
|
||||
expect(resData).toBe('1197161814');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,135 @@
|
||||
import config from "../../config";
|
||||
import { CacheInterface, LoggerInterface, WeiboStatus, WeiboUserData } from "../../types";
|
||||
import { logger } from "../logger";
|
||||
import { createDetailAPI, GetWeiboDetailFunc } from "./api/detailAPI";
|
||||
import { createDomainAPI, DomainNotFoundError, GetUIDByDomainFunc } from "./api/domainAPI";
|
||||
import { createIndexAPI, GetIndexUserInfoFunc, GetWeiboContentListFunc, UserNotFoundError } from "./api/indexAPI";
|
||||
import { createLongTextAPI, GetWeiboLongTextFunc } from "./api/longTextAPI";
|
||||
|
||||
export {
|
||||
DomainNotFoundError,
|
||||
UserNotFoundError,
|
||||
};
|
||||
|
||||
export class WeiboData {
|
||||
private cache: CacheInterface;
|
||||
private logger: LoggerInterface;
|
||||
private getIndexUserInfo: GetIndexUserInfoFunc;
|
||||
private getWeiboContentList: GetWeiboContentListFunc;
|
||||
private getWeiboDetail: GetWeiboDetailFunc;
|
||||
private getWeiboLongText: GetWeiboLongTextFunc;
|
||||
private getUIDByDomain: GetUIDByDomainFunc;
|
||||
|
||||
constructor(cache: CacheInterface, log: LoggerInterface = logger) {
|
||||
this.cache = cache;
|
||||
this.logger = log;
|
||||
const { getIndexUserInfo, getWeiboContentList } = createIndexAPI();
|
||||
const { getWeiboDetail } = createDetailAPI();
|
||||
const { getWeiboLongText } = createLongTextAPI();
|
||||
const { getUIDByDomain } = createDomainAPI();
|
||||
// bind to this
|
||||
Object.assign(this, {
|
||||
getIndexUserInfo,
|
||||
getWeiboContentList,
|
||||
getWeiboDetail,
|
||||
getWeiboLongText,
|
||||
getUIDByDomain,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* get user's weibo
|
||||
*/
|
||||
fetchUserLatestWeibo = async (uid: string) => {
|
||||
const indexInfo = await this.cache.memo(() => this.getIndexUserInfo(uid), `info-${uid}`, config.cacheTTL.apiIndexInfo);
|
||||
const { containerId } = indexInfo;
|
||||
const statusList = await this.cache.memo(async () => {
|
||||
const wbList = await this.getWeiboContentList(uid, containerId);
|
||||
return await Promise.all(
|
||||
wbList.map(status => this.fillStatusWithLongText(status))
|
||||
);
|
||||
}, `list-${uid}`, config.cacheTTL.apiStatusList);
|
||||
|
||||
return {
|
||||
...indexInfo,
|
||||
statusList,
|
||||
} as WeiboUserData;
|
||||
};
|
||||
|
||||
/**
|
||||
* allow failure
|
||||
*/
|
||||
fillStatusWithLongText = async (status: WeiboStatus) => {
|
||||
let newStatus = status;
|
||||
try {
|
||||
if (status.isLongText) {
|
||||
try {
|
||||
const longTextContent = await this.cache.memo(
|
||||
() => this.getWeiboLongText(status.id),
|
||||
`long-${status.id}`,
|
||||
config.cacheTTL.apiLongText,
|
||||
);
|
||||
newStatus = {
|
||||
...status,
|
||||
text: longTextContent
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error(error, `uid: ${status?.user?.id}, status: ${status.id}`);
|
||||
// fallback to detail
|
||||
newStatus = await this.cache.memo(
|
||||
() => this.getWeiboDetail(status.id),
|
||||
`dt-${status.id}`,
|
||||
config.cacheTTL.apiDetail,
|
||||
);
|
||||
}
|
||||
}
|
||||
// 转发的微博全文
|
||||
if (status.retweeted_status) {
|
||||
newStatus = {
|
||||
...status,
|
||||
retweeted_status: await this.fillStatusWithLongText(status.retweeted_status),
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(error, `uid: ${status?.user?.id}, status: ${status.id}`);
|
||||
}
|
||||
return newStatus;
|
||||
};
|
||||
|
||||
/**
|
||||
* domain -> uid
|
||||
*/
|
||||
fetchUIDByDomain = async (domain: string) => this.getUIDByDomain(domain);
|
||||
}
|
||||
|
||||
export const statusToHTML = (status: WeiboStatus) => {
|
||||
let tempHTML = status.text;
|
||||
// 表情转文字
|
||||
tempHTML = tempHTML.replace(/<span class="url-icon"><img alt="?(.*?)"? src=".*?" style="width:1em; height:1em;".*?\/><\/span>/g, '$1');
|
||||
// 去掉外链图标
|
||||
tempHTML = tempHTML.replace(/<span class='url-icon'><img.*?><\/span>/g, '');
|
||||
|
||||
// 转发的微博
|
||||
if (status.retweeted_status) {
|
||||
tempHTML += "<br><br>";
|
||||
// 可能有转发的微博被删除的情况
|
||||
if (status.retweeted_status.user) {
|
||||
tempHTML += '<div style="border-left: 3px solid gray; padding-left: 1em;">' +
|
||||
'转发 <a href="https://weibo.com/' + status.retweeted_status.user.id + '" target="_blank">@' + status.retweeted_status.user.screen_name + '</a>: ' +
|
||||
statusToHTML(status.retweeted_status) +
|
||||
'</div>';
|
||||
}
|
||||
}
|
||||
|
||||
// 微博配图
|
||||
if (status.pics) {
|
||||
status.pics.forEach(function (item) {
|
||||
tempHTML += "<br><br>";
|
||||
const url = config.imageCache ? (config.imageCache + encodeURIComponent(item.large.url)) : item.large.url;
|
||||
const largeUrl = config.imageCache ? (config.imageCache + encodeURIComponent(item.large.url)) : item.large.url;
|
||||
tempHTML += '<a href="' + largeUrl + '" target="_blank"><img src="' + url+ '"></a>';
|
||||
});
|
||||
}
|
||||
|
||||
return tempHTML;
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { Tracer } from "tracer";
|
||||
import { WeiboData } from "./modules/weibo/weibo";
|
||||
|
||||
export interface RSSKoaContext {
|
||||
cache: CacheInterface;
|
||||
weibo: WeiboData;
|
||||
}
|
||||
|
||||
export interface RSSKoaState {
|
||||
// cache hit
|
||||
hit: 0 | 1;
|
||||
}
|
||||
|
||||
export type LoggerInterface = Tracer.Logger<string>;
|
||||
|
||||
export interface CacheInterface {
|
||||
set: (key: string, value: any, expire: number) => Promise<void>;
|
||||
get: (key: string) => any;
|
||||
memo: <T>(cb: () => T, key: string, expire: number) => Promise<Awaited<T>>;
|
||||
}
|
||||
|
||||
export interface WeiboStatus {
|
||||
// eg: '4851725594528288'
|
||||
id: string;
|
||||
// eg: '4851725594528288'
|
||||
mid: string;
|
||||
// eg: 'MlHCnj00U'
|
||||
bid: string;
|
||||
// time
|
||||
created_at: string;
|
||||
// text
|
||||
text: string;
|
||||
isLongText: boolean;
|
||||
user: {
|
||||
id: number,
|
||||
screen_name: string,
|
||||
profile_url: string;
|
||||
description: string;
|
||||
// more...
|
||||
[x: string]: any;
|
||||
},
|
||||
// pics
|
||||
pic_ids: string[];
|
||||
thumbnail_pic: string;
|
||||
bmiddle_pic: string;
|
||||
original_pic: string;
|
||||
pics: {
|
||||
pid: string,
|
||||
url: string,
|
||||
size: 'orj360' | 'large',
|
||||
large: {
|
||||
size: 'orj360' | 'large',
|
||||
url: string,
|
||||
}
|
||||
}[];
|
||||
// video and other data
|
||||
page_info?: {
|
||||
type: 'video' | 'search_topic';
|
||||
// eg: 'http://t.cn/A6K3ITkN'
|
||||
url_ori: string;
|
||||
// eg: '搜狐新闻的微博视频'
|
||||
page_title: string;
|
||||
// eg: '2022触动瞬间'
|
||||
title: string;
|
||||
// eg: '搜狐新闻的微博视频'
|
||||
content1: string;
|
||||
content2: string;
|
||||
// more...
|
||||
[x: string]: any;
|
||||
};
|
||||
retweeted_status?: WeiboStatus;
|
||||
[x: string]: any;
|
||||
}
|
||||
|
||||
export interface WeiboUserData {
|
||||
uid: string,
|
||||
screenName: string,
|
||||
description: string,
|
||||
containerId?: string,
|
||||
statusList?: WeiboStatus[],
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
|
||||
export const normalizePort = (val: string) => {
|
||||
const port = parseInt(val, 10);
|
||||
if (isNaN(port)) {
|
||||
// named pipe
|
||||
return val;
|
||||
}
|
||||
if (port >= 0) {
|
||||
// port number
|
||||
return port;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export const waitMs = (ms: number) => {
|
||||
return new Promise(resolve => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist/",
|
||||
"rootDir": "./src/",
|
||||
"target": "es2018",
|
||||
"module": "commonjs",
|
||||
"moduleResolution": "node",
|
||||
"esModuleInterop": true,
|
||||
"experimentalDecorators": true,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user