0.3.0原版

src/app.ts

@@ -0,0 +1,53 @@
+import Koa from 'koa';
+import Router from '@koa/router';
+import serve from 'koa-static';
+import path from 'path';
+import config from './config';
+import { logger } from "./modules/logger";
+import { normalizePort } from './utils';
+import { registerRoutes } from './modules/routes';
+import { RSSKoaContext, RSSKoaState } from './types';
+import { LevelCache } from './modules/cache';
+import { WeiboData } from './modules/weibo/weibo';
+
+const koaApp = new Koa<RSSKoaState, RSSKoaContext>();
+const initApp = () => {
+  const router = new Router<RSSKoaState, RSSKoaContext>();
+  registerRoutes(router);
+
+  // levelCache
+  const cache = LevelCache.getInstance(path.join(config.rootDir, 'data'), logger);
+  cache.startScheduleCleanJob();
+
+  // weibo
+  const weiboData = new WeiboData(cache, logger);
+
+  // enable X-Forwarded-For
+  koaApp.proxy = true;
+
+  // init middleware
+  return koaApp
+    .use(async (ctx, next) => {
+      // duration log
+      const startTime = Date.now();
+      logger.debug(`${ctx.req.method} ${ctx.originalUrl} ${ctx.ip}`);
+      await next();
+      const duration = Date.now() - startTime;
+      logger.info(`[${ctx.status}] ${ctx.req.method} ${ctx.originalUrl} ${ctx.ip} hit: ${ctx.state.hit || 0} ${duration}ms`);
+    })
+    .use(async (ctx, next) => {
+      ctx.cache = cache;
+      ctx.weibo = weiboData;
+      await next();
+    })
+    .use(serve(config.rootDir + '/public'))
+    .use(router.routes());
+};
+
+// start service
+initApp();
+const port = normalizePort(process.env.PORT || config.port);
+koaApp.listen(port, () => {
+  logger.info(`weibo-rss start`);
+  logger.info(`Listening http://0.0.0.0:${port}`);
+});

src/config.ts

@@ -0,0 +1,45 @@
+/**
+ * 配置文件加载相关
+ */
+import { existsSync } from "fs";
+import { logger } from "./modules/logger";
+
+const rootDir = `${__dirname}/../`;
+let customConfig = {};
+
+// 加载自定义配置
+if (existsSync(`${rootDir}/config.js`)) {
+  try {
+    customConfig = require('../config');
+  } catch (err) {
+    logger.error('custom config load failed,', err);
+  }
+}
+
+// 默认配置
+const defaultConfig = {
+  // 程序监听的 TCP 端口，也可以在环境变量指定
+  port: '3000',
+  // 输出 rss feed 文本的 ttl 字段（分钟为单位）
+  rssTTL: 15,
+  // 真正的缓存配置，具体关注代码（秒为单位）
+  cacheTTL: {
+    rssXml: 15 * 60,
+    apiStatusList: 15 * 60,
+    apiIndexInfo: 1 * 24 * 60 * 60,
+    apiLongText: 7 * 24 * 60 * 60,
+    apiDetail: 7 * 24 * 60 * 60,
+    apiDomain: 7 * 24 * 60 * 60,
+  },
+  // 图片缓存代理
+  imageCache: 'https://image.baidu.com/search/down?url=',
+};
+
+/**
+ * 自定义配置
+ */
+export default {
+  ...defaultConfig,
+  ...customConfig,
+  rootDir, // 站点根路径
+};

src/modules/cache.ts

@@ -0,0 +1,158 @@
+import LevelDOWN from "leveldown";
+import levelUp, { LevelUp } from "levelup";
+import { scheduleJob } from "node-schedule";
+import path from "path";
+import { CacheInterface, LoggerInterface } from "../types";
+import { logger } from "./logger";
+
+// 缓存条目的结构
+export interface CacheObject {
+  created: number; // 缓存设置时的时间戳（毫秒）
+  expire: boolean | number; // 有效期（秒），falsy 为不清理
+  value: any;
+};
+
+// levelup 的错误类型
+// https://github.com/Level/errors/blob/f5e5e406a5e325a8a62eb4d1135fa3d010816f8b/errors.js
+export interface NotFoundError extends Error {
+  notFound: boolean;
+}
+
+// 默认缓存目录
+const DB_FOLDER = 'rss-data';
+
+/**
+ * LevelDB 实现的文件系统缓存
+ */
+export class LevelCache implements CacheInterface {
+  private instance: LevelUp;
+  private logger: LoggerInterface;
+
+  constructor(dbPath: string, logger: LoggerInterface) {
+    this.instance = levelUp(LevelDOWN(dbPath));
+    this.logger = logger;
+  }
+
+  static instance: LevelCache = null;
+
+  static getInstance(dataBaseDir: string, log: LoggerInterface = logger) {
+    if (!LevelCache.instance) {
+      LevelCache.instance = new LevelCache(path.join(dataBaseDir, DB_FOLDER), log);
+    }
+    return LevelCache.instance;
+  }
+
+  /**
+   * 设置缓存
+   * @param key 需缓存的 key
+   * @param value key 对应的值
+   * @param expire 过期时间（单位秒）
+   */
+  set(key: string, value: any, expire: number = 0) {
+    this.logger.debug(`[cache] set ${key}`);
+    return new Promise<void>((resolve, reject) => {
+      const data: CacheObject = {
+        created: Date.now(),
+        expire: !!expire,
+        value,
+      };
+      if (expire) {
+        data.expire = expire;
+      }
+      this.instance.put(key, JSON.stringify(data), (err) => {
+        if (err) return reject(err);
+        return resolve();
+      });
+    });
+  }
+
+  /**
+   * 获取缓存的值
+   * @param key 缓存对象的 key
+   */
+  get(key: string) {
+    return new Promise((resolve, reject) => {
+      this.instance.get(key, (err: NotFoundError, value) => {
+        if (err) {
+          if (err.notFound) {
+            this.logger.debug(`[cache] get ${key} notFound`);
+            return resolve(null);
+          }
+          this.logger.error(`[cache] get ${key} error`, err);
+          return reject(err);
+        }
+        try {
+          const data = JSON.parse(String(value)) as CacheObject;
+          if (this.checkExpired(data)) {
+            this.logger.debug(`[cache] get ${key} expired`);
+            // 过期缓存交给定时任务清理
+            return resolve(null);
+          } else {
+            this.logger.debug(`[cache] get ${key}`);
+            return resolve(data.value);
+          }
+        } catch (error) {
+          this.logger.error(`[cache] parse ${key} error`, error);
+          return reject(error);
+        }
+      });
+    });
+  }
+
+  /**
+   * memo in cache
+   */
+  async memo<T>(cb: () => T, key: string, expire = 0): Promise<Awaited<T>> {
+    const cacheResp = await this.get(key);
+    if (cacheResp) {
+      return cacheResp as Awaited<T>;
+    }
+    const res = await cb();
+    this.set(key, res, expire);
+    return res;
+  }
+
+  /**
+   * 定期清理过期缓存
+   */
+  startScheduleCleanJob(rule: Parameters<typeof scheduleJob>[0] = "0 30 2 * * *") {
+    // 每日凌晨两点半开始处理
+    return scheduleJob(rule, () => {
+      this.logger.info("[cache] cleaning start");
+      let total = 0,
+        deleted = 0;
+      this.instance
+        .createReadStream()
+        .on("data", (item) => {
+          total++;
+          try {
+            const data = JSON.parse(item.value.toString()) as CacheObject;
+            if (this.checkExpired(data)) {
+              deleted++;
+              setTimeout(() => {
+                this.instance.del(item.key, (err) => {
+                  if (err) {
+                    this.logger.error(`[cache] delete err key: ${item.key}`, err);
+                  }
+                });
+              }, 0);
+            }
+          } catch (err) {
+            this.logger.error("[cache] parse error", err);
+          }
+        })
+        .on("end", () => {
+          this.logger.info(
+            "[cache] cleaning finished, total cnt:",
+            total,
+            "deleted cnt:",
+            deleted
+          );
+        });
+    });
+  }
+
+  private checkExpired(cacheItem: CacheObject) {
+    return cacheItem.expire && Date.now() - cacheItem.created > +cacheItem.expire * 1000;
+  }
+}

src/modules/logger.ts

@@ -0,0 +1,11 @@
+/**
+ * 日志模块，生成对应格式日志输出到控制台
+ */
+import * as tracer from 'tracer';
+
+const debug = process.env.DEBUG;
+
+export const logger = tracer.colorConsole({
+  format : "[{{timestamp}}] [{{title}}] {{message}}",
+  level: debug ? 'debug' : 'info',
+});

src/modules/routes.ts

@@ -0,0 +1,134 @@
+/**
+ * URL 路由分发
+ */
+import Router from '@koa/router';
+import NodeRSS from 'rss';
+import { RSSKoaContext, RSSKoaState } from '../types';
+import config from '../config';
+import { DomainNotFoundError, statusToHTML, UserNotFoundError } from './weibo/weibo';
+import { ThrottledError } from './throttler';
+import { logger } from './logger';
+
+export class UidInvalidError extends Error {
+  constructor(uid: string) {
+    super(`uid: ${uid}`);
+  }
+}
+
+export class DomainInvalidError extends Error {
+  constructor(domain: string) {
+    super(`domain: ${domain}`);
+  }
+}
+
+export const registerRoutes = (
+  router: Router<RSSKoaState, RSSKoaContext>
+) => {
+  router.get('/rss/user/:id', async (ctx) => {
+    const uid = ctx.params['id'];
+    try {
+      // check uid format
+      if (!/^[0-9]{10}$/.test(uid)) {
+        throw new UidInvalidError(uid);
+      }
+
+      // get data
+      let cacheMiss = false;
+      const xmlData = await ctx.cache.memo(async () => {
+        const weiboData = await ctx.weibo.fetchUserLatestWeibo(uid);
+        if (weiboData) {
+          // basic info
+          const feed = new NodeRSS({
+            site_url: "https://weibo.com/" + uid,
+            feed_url: '',
+            title: weiboData.screenName + '的微博',
+            description: weiboData.description,
+            generator: 'https://github.com/zgq354/weibo-rss',
+            ttl: config.rssTTL,
+          });
+          // item
+          weiboData.statusList?.forEach((status) => {
+            if (!status) return;
+            feed.item({
+              title: status.status_title || (status.text ? status.text.replace(/<[^>]+>/g, '').replace(/[\n]/g, '').substr(0, 25) : null),
+              description: statusToHTML(status),
+              url: 'https://weibo.com/' + uid + '/' + status.bid,
+              date: new Date(status.created_at),
+            });
+          });
+          cacheMiss = true;
+          return feed.xml();
+        }
+      }, `xml-${uid}`, config.cacheTTL.rssXml);
+
+      // send data
+      ctx.set('Content-Type', 'text/xml');
+      ctx.body = xmlData;
+
+      // mark hit cache
+      ctx.state.hit = cacheMiss ? 0 : 1;
+    } catch (error) {
+      if (error instanceof UidInvalidError) {
+        ctx.status = 404;
+        ctx.body = `找不到用户，传入 UID 格式有误。uid: ${uid}`;
+        return;
+      }
+      if (error instanceof UserNotFoundError) {
+        ctx.status = 404;
+        ctx.body = `找不到用户，可能用户仅登录可见，不支持订阅。可以通过打开 https://m.weibo.cn/u/:uid 验证（<a href="https://m.weibo.cn/u/${uid}" target="_blank">uid: ${uid}</a>）`;
+        return;
+      }
+      if (error instanceof ThrottledError) {
+        ctx.status = 503;
+        ctx.body = `暂时无法拉取到数据，请稍后再试。uid: ${uid}`;
+        return;
+      }
+      ctx.status = 500;
+      ctx.body = `未知错误，需管理员检查日志。uid: ${uid}`;
+      logger.error(error);
+    }
+  });
+
+  router.get('/api/domain2uid', async (ctx) => {
+    const domain = ctx.request.query['domain'] as string;
+    try {
+      // verify
+      if (!domain || !/^[A-Za-z0-9]{3,20}$/.test(domain)) {
+        throw new DomainInvalidError(domain);
+      }
+      // start fetching
+      let cacheMiss = false;
+      const uid = await ctx.cache.memo(
+        () => {
+          cacheMiss = true;
+          return ctx.weibo.fetchUIDByDomain(domain);
+        },
+        `dm-${domain}`,
+        config.cacheTTL.apiDomain,
+      );
+      logger.debug(`domain: ${domain}, uid: ${uid}`);
+      ctx.body = {
+        success: true,
+        uid
+      };
+
+      // mark hit cache
+      ctx.state.hit = cacheMiss ? 0 : 1;
+    } catch (error) {
+      if (error instanceof DomainInvalidError || error instanceof DomainNotFoundError) {
+        ctx.status = 404;
+        ctx.body = {
+          success: false,
+          msg: '找不到用户，可能是地址格式不正确',
+        };
+        return;
+      }
+      logger.error(error);
+      ctx.status = 500;
+      ctx.body = {
+        success: false,
+        msg: '获取数据时发生了错误'
+      };
+    }
+  })
+};

src/modules/throttler.ts

@@ -0,0 +1,52 @@
+import Queue from "np-queue";
+import { LoggerInterface } from "../types";
+import { logger } from "./logger";
+
+export class ThrottledError extends Error {
+  constructor() {
+    super();
+  }
+}
+
+/**
+ * Limit the request frequency
+ */
+export class Throttler {
+  name: string = '';
+  queue: Queue;
+  enable: boolean;
+  lastUpdateTime: number;
+  retryDelayMs: number;
+  logger: LoggerInterface;
+
+  constructor(name = '', log: LoggerInterface = logger) {
+    this.name = name;
+    this.logger = log;
+    this.queue = new Queue({ concurrency: 1 });
+    this.retryDelayMs = 600000;
+    this.enable = true;
+    this.lastUpdateTime = Date.now();
+  }
+
+  checkFuncAvailable = () => {
+    if (!this.enable && Date.now() - this.lastUpdateTime > this.retryDelayMs) {
+      this.enable = true;
+      this.logger.info(`[Throttled] reenabled ${this.name}`);
+    }
+    return this.enable;
+  };
+
+  disableFunc = () => {
+    this.enable = false;
+    this.lastUpdateTime = Date.now();
+    this.logger.info(`[Throttled] disabled ${this.name}`);
+    return Promise.reject(new ThrottledError());
+  };
+
+  runFunc = <T = any>(callback: (disable: () => Promise<void>) => any): Promise<T> => {
+    if (!this.checkFuncAvailable()) {
+      return Promise.reject(new ThrottledError());
+    }
+    return this.queue.add(async () => callback(this.disableFunc));
+  };
+}

src/modules/weibo/api/common.ts

@@ -0,0 +1,12 @@
+import { AxiosError } from "axios";
+
+export const TIME_OUT = 3000;
+export const MOCK_UA = 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36';
+
+export const handleForbiddenErr = (err: AxiosError, cb: () => Promise<void>) => {
+  if (err.response && [418, 403].includes(err.response.status)) {
+    return cb();
+  } else {
+    return Promise.reject(err);
+  }
+}

src/modules/weibo/api/detailAPI.spec.ts

@@ -0,0 +1,8 @@
+// TODO: add more tests
+import { describe, expect, test } from "@jest/globals";
+
+describe('Weibo API: detailAPI', () => {
+  test('TODO: add more tests', async () => {
+    expect(1).toBe(1);
+  });
+});

src/modules/weibo/api/detailAPI.ts

@@ -0,0 +1,35 @@
+import { Throttler } from "../../throttler";
+import Axios from "axios";
+import { Agent } from "https";
+import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
+import { logger } from "../../logger";
+import { waitMs } from "../../../utils";
+
+export const createDetailAPI = () => {
+  const runner = new Throttler('detail');
+  const httpsAgent = new Agent({ keepAlive: true });
+  const axiosInstance = Axios.create({
+    timeout: TIME_OUT,
+    httpsAgent
+  });
+  return {
+    getWeiboDetail: (id: string) => runner.runFunc(async (disable) => {
+      logger.debug(`[getDetail] ${id}`);
+      await waitMs(Math.floor(Math.random() * 100));
+      return await axiosInstance({
+        method: 'get',
+        url: `https://m.weibo.cn/statuses/show?id=${id}`,
+        headers: {
+          'Mweibo-Pwa': 1,
+          'Referer': `https://m.weibo.cn/detail/${id}`,
+          'User-Agent': MOCK_UA,
+          'X-Requested-With': 'XMLHttpRequest'
+        },
+      }).then(({ data }) => {
+        return data.data;
+      }).catch(err => handleForbiddenErr(err, disable));
+    }),
+  };
+}
+
+export type GetWeiboDetailFunc = ReturnType<typeof createDetailAPI>['getWeiboDetail'];

src/modules/weibo/api/domainAPI.spec.ts

@@ -0,0 +1,27 @@
+import { describe, expect, test } from "@jest/globals";
+import { createDomainAPI } from "./domainAPI";
+
+const testDomainMapList = [
+  {
+    domain: 'kaifulee',
+    uid: '1197161814',
+  },
+  {
+    domain: 'taobao',
+    uid: '1682454721',
+  },
+  {
+    domain: 'tmall',
+    uid: '1768198384',
+  },
+];
+
+describe('Domain API: domain to uid', () => {
+  test('basic domain format', async () => {
+    const { getUIDByDomain } = createDomainAPI();
+    testDomainMapList.forEach(async (data) => {
+      const resData = await getUIDByDomain(data.domain);
+      expect(resData).toBe(data.uid);
+    });
+  });
+});

src/modules/weibo/api/domainAPI.ts

@@ -0,0 +1,41 @@
+import { Throttler } from "../../throttler";
+import Axios from "axios";
+import { Agent } from "https";
+import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
+import { logger } from "../../logger";
+import { waitMs } from "../../../utils";
+
+export class DomainNotFoundError extends Error {
+  constructor(domain: string) {
+    super(`domain: ${domain}`);
+  }
+}
+
+export const createDomainAPI = () => {
+  const runner = new Throttler('domain');
+  const httpsAgent = new Agent({ keepAlive: true });
+  const axiosInstance = Axios.create({
+    timeout: TIME_OUT,
+    httpsAgent
+  });
+
+  return {
+    getUIDByDomain: (domain: string) => runner.runFunc<string>(async (disable) => {
+      logger.debug(`[domain] convert ${domain}`);
+      await waitMs(Math.floor(Math.random() * 100));
+      return await axiosInstance.get(`https://m.weibo.cn/${domain}?&jumpfrom=weibocom`, {
+        headers: {
+          'User-Agent': MOCK_UA,
+        },
+      }).then(res => {
+        const uid = res.request.path.split("/u/")[1] as string;
+        if (!uid) {
+          throw new DomainNotFoundError(domain);
+        }
+        return uid;
+      }).catch(err => handleForbiddenErr(err, disable));
+    }),
+  };
+};
+
+export type GetUIDByDomainFunc = ReturnType<typeof createDomainAPI>['getUIDByDomain'];

src/modules/weibo/api/indexAPI.spec.ts

@@ -0,0 +1,8 @@
+// TODO: add more tests
+import { describe, expect, test } from "@jest/globals";
+
+describe('Weibo API: indexAPI', () => {
+  test('TODO: add more tests', async () => {
+    expect(1).toBe(1);
+  });
+});

src/modules/weibo/api/indexAPI.ts

@@ -0,0 +1,86 @@
+import { Throttler } from "../../throttler";
+import Axios from "axios";
+import { Agent } from "https";
+import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
+import { WeiboStatus, WeiboUserData } from "../../../types";
+import { logger } from "../../logger";
+import { waitMs } from "../../../utils";
+
+export class UserNotFoundError extends Error {
+  constructor(uid: string) {
+    super(`uid: ${uid}`);
+  }
+}
+
+export const createIndexAPI = () => {
+  const runner = new Throttler('index');
+  const httpsAgent = new Agent({ keepAlive: true });
+  const axiosInstance = Axios.create({
+    timeout: TIME_OUT,
+    httpsAgent
+  });
+
+  return {
+    getIndexUserInfo: (uid: string) => runner.runFunc<WeiboUserData>(async (disable) => {
+      const resHeaders = await axiosInstance({
+        "method": "POST",
+        "url": "https://visitor.passport.weibo.cn/visitor/genvisitor2",
+        "headers": {
+          "Content-Type": "application/x-www-form-urlencoded",
+          "User-Agent": MOCK_UA,
+        },
+        "data": {
+          "cb": "visitor_gray_callback",
+          tid: '',
+          from: 'weibo'
+        }
+      }).then(res => res.headers)
+      axiosInstance.defaults.headers.common['Cookie'] = resHeaders['set-cookie'].filter(cookie => cookie.startsWith('SUB')).map(cookie => cookie.split(';')[0]).join(';');
+      logger.debug(`[getInfo] ${uid}`);
+      await waitMs(Math.floor(Math.random() * 100));
+      return await axiosInstance({
+        method: 'get',
+        url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
+        headers: {
+          'Mweibo-Pwa': 1,
+          'Referer': `https://m.weibo.cn/u/${uid}`,
+          'User-Agent': MOCK_UA,
+          'X-Requested-With': 'XMLHttpRequest'
+        }
+      }).then(({ data }) => {
+        if (data.ok !== 1) {
+          return Promise.reject(new UserNotFoundError(uid));
+        }
+        const result = {
+          uid,
+          screenName: data.data.userInfo.screen_name,
+          description: data.data.userInfo.description,
+          containerId: data.data.tabsInfo.tabs[1].containerid,
+        };
+        return result;
+      }).catch(err => handleForbiddenErr(err, disable));
+    }),
+    getWeiboContentList: (uid: string, containerId: string) => runner.runFunc<WeiboStatus[]>(async (disable) => {
+      logger.debug(`[getContList] ${uid} ${containerId}`);
+      await waitMs(Math.floor(Math.random() * 100));
+      return await axiosInstance({
+        method: 'get',
+        url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}&containerid=${containerId}`,
+        headers: {
+          'Mweibo-Pwa': 1,
+          'Referer': `https://m.weibo.cn/u/${uid}`,
+          'User-Agent': MOCK_UA,
+          'X-Requested-With': 'XMLHttpRequest'
+        }
+      }).then(({ data }) => {
+        return data.data.cards
+          .filter(item => item.mblog)
+          .map(item => item.mblog);
+      }).catch(err => handleForbiddenErr(err, disable));
+    }),
+  };
+};
+
+export type GetIndexUserInfoFunc = ReturnType<typeof createIndexAPI>['getIndexUserInfo'];
+
+export type GetWeiboContentListFunc = ReturnType<typeof createIndexAPI>['getWeiboContentList'];

src/modules/weibo/api/longTextAPI.spec.ts

@@ -0,0 +1,20 @@
+import { describe, expect, test } from "@jest/globals";
+import { createLongTextAPI } from "./longTextAPI";
+
+const testDataList = [{
+  id: '5093426468489917',
+  text: '中微子是宇宙形成之初就存在的最古老也最原始的基本粒子',
+}, {
+  id: '5093698779485067',
+  text: '无论如何，「AI God」的拍卖再次引发了人们对传统艺术与数字艺术的思考',
+}];
+
+describe ('Weibo API: longTextAPI', () => {
+  test('fetch long text', async () => {
+    const { getWeiboLongText } = createLongTextAPI();
+    for (const data of testDataList) {
+      const resData = await getWeiboLongText(data.id);
+      expect(resData).toContain(data.text);
+    }
+  });
+});

src/modules/weibo/api/longTextAPI.ts

@@ -0,0 +1,38 @@
+import { Throttler } from "../../throttler";
+import Axios from "axios";
+import { Agent } from "https";
+import { handleForbiddenErr, MOCK_UA, TIME_OUT } from "./common";
+import { logger } from "../../logger";
+import { waitMs } from "../../../utils";
+
+export const createLongTextAPI = () => {
+  const runner = new Throttler('longText');
+  const httpsAgent = new Agent({ keepAlive: true });
+  const axiosInstance = Axios.create({
+    timeout: TIME_OUT,
+    httpsAgent
+  });
+  return {
+    getWeiboLongText: (id: string) => runner.runFunc<string>(async (disable) => {
+      logger.debug(`[longText] ${id}`);
+      await waitMs(Math.floor(Math.random() * 100));
+      return await axiosInstance({
+        method: 'get',
+        url: `https://m.weibo.cn/statuses/extend?id=${id}`,
+        headers: {
+          'Mweibo-Pwa': 1,
+          'Referer': `https://m.weibo.cn/detail/${id}`,
+          'User-Agent': MOCK_UA,
+          'X-Requested-With': 'XMLHttpRequest'
+        },
+      }).then(({ data }) => {
+        if (!data.data) {
+          throw new Error(JSON.stringify(data));
+        }
+        return data.data.longTextContent;
+      }).catch(err => handleForbiddenErr(err, disable));
+    }),
+  };
+};
+
+export type GetWeiboLongTextFunc = ReturnType<typeof createLongTextAPI>['getWeiboLongText'];

src/modules/weibo/weibo.spec.ts

@@ -0,0 +1,38 @@
+import { describe, expect, test } from '@jest/globals';
+import { WeiboData } from "./weibo";
+import { WeiboStatus } from '../../types';
+
+const wbData = new WeiboData({
+  set: async () => null,
+  get: async () => null,
+  memo: async <T>(cb: () => T): Promise<Awaited<T>> => await cb(),
+});
+
+// TODO: more unit test about weibo data
+describe('Weibo Data: user weibo profile and list', () => {
+  const TEST_UID = '5890672121';
+
+  // 获取用户名 + 微博列表
+  test('fetch user weibo profile and list success', async () => {
+    const resData = await wbData.fetchUserLatestWeibo(TEST_UID);
+    expect(resData.uid).toBe(TEST_UID);
+    expect(resData.screenName).toBe('搜狐新闻');
+    expect(resData.statusList).toBeDefined();
+  });
+
+  // 长文本的填充
+  test('fetch weibo with long text', async () => {
+    const resData = await wbData.fillStatusWithLongText({
+      id: '5093426468489917',
+      isLongText: true,
+    } as WeiboStatus);
+    expect(resData.text).toContain('中微子是宇宙形成之初就存在的最古老也最原始的基本粒子');
+  });
+});
+
+describe('Weibo Data: domain to uid', () => {
+  test('basic domain format', async () => {
+    const resData = await wbData.fetchUIDByDomain('kaifulee');
+    expect(resData).toBe('1197161814');
+  });
+});

src/modules/weibo/weibo.ts

@@ -0,0 +1,135 @@
+import config from "../../config";
+import { CacheInterface, LoggerInterface, WeiboStatus, WeiboUserData } from "../../types";
+import { logger } from "../logger";
+import { createDetailAPI, GetWeiboDetailFunc } from "./api/detailAPI";
+import { createDomainAPI, DomainNotFoundError, GetUIDByDomainFunc } from "./api/domainAPI";
+import { createIndexAPI, GetIndexUserInfoFunc, GetWeiboContentListFunc, UserNotFoundError } from "./api/indexAPI";
+import { createLongTextAPI, GetWeiboLongTextFunc } from "./api/longTextAPI";
+
+export {
+  DomainNotFoundError,
+  UserNotFoundError,
+};
+
+export class WeiboData {
+  private cache: CacheInterface;
+  private logger: LoggerInterface;
+  private getIndexUserInfo: GetIndexUserInfoFunc;
+  private getWeiboContentList: GetWeiboContentListFunc;
+  private getWeiboDetail: GetWeiboDetailFunc;
+  private getWeiboLongText: GetWeiboLongTextFunc;
+  private getUIDByDomain: GetUIDByDomainFunc;
+
+  constructor(cache: CacheInterface, log: LoggerInterface = logger) {
+    this.cache = cache;
+    this.logger = log;
+    const { getIndexUserInfo, getWeiboContentList } = createIndexAPI();
+    const { getWeiboDetail } = createDetailAPI();
+    const { getWeiboLongText } = createLongTextAPI();
+    const { getUIDByDomain } = createDomainAPI();
+    // bind to this
+    Object.assign(this, {
+      getIndexUserInfo,
+      getWeiboContentList,
+      getWeiboDetail,
+      getWeiboLongText,
+      getUIDByDomain,
+    });
+  }
+
+  /**
+   * get user's weibo
+   */
+  fetchUserLatestWeibo = async (uid: string) => {
+    const indexInfo = await this.cache.memo(() => this.getIndexUserInfo(uid), `info-${uid}`, config.cacheTTL.apiIndexInfo);
+    const { containerId } = indexInfo;
+    const statusList = await this.cache.memo(async () => {
+      const wbList = await this.getWeiboContentList(uid, containerId);
+      return await Promise.all(
+        wbList.map(status => this.fillStatusWithLongText(status))
+      );
+    }, `list-${uid}`, config.cacheTTL.apiStatusList);
+
+    return {
+      ...indexInfo,
+      statusList,
+    } as WeiboUserData;
+  };
+
+  /**
+   * allow failure
+   */
+  fillStatusWithLongText = async (status: WeiboStatus) => {
+    let newStatus = status;
+    try {
+      if (status.isLongText) {
+        try {
+          const longTextContent = await this.cache.memo(
+            () => this.getWeiboLongText(status.id),
+            `long-${status.id}`,
+            config.cacheTTL.apiLongText,
+          );
+          newStatus = {
+            ...status,
+            text: longTextContent
+          };
+        } catch (error) {
+          logger.error(error, `uid: ${status?.user?.id}, status: ${status.id}`);
+          // fallback to detail
+          newStatus = await this.cache.memo(
+            () => this.getWeiboDetail(status.id),
+            `dt-${status.id}`,
+            config.cacheTTL.apiDetail,
+          );
+        }
+      }
+      // 转发的微博全文
+      if (status.retweeted_status) {
+        newStatus = {
+          ...status,
+          retweeted_status: await this.fillStatusWithLongText(status.retweeted_status),
+        }
+      }
+    } catch (error) {
+      logger.error(error, `uid: ${status?.user?.id}, status: ${status.id}`);
+    }
+    return newStatus;
+  };
+
+  /**
+   * domain -> uid
+   */
+  fetchUIDByDomain = async (domain: string) => this.getUIDByDomain(domain);
+}
+
+export const statusToHTML = (status: WeiboStatus) => {
+  let tempHTML = status.text;
+  // 表情转文字
+  tempHTML = tempHTML.replace(/<span class="url-icon"><img alt="?(.*?)"? src=".*?" style="width:1em; height:1em;".*?\/><\/span>/g, '$1');
+  // 去掉外链图标
+  tempHTML = tempHTML.replace(/<span class='url-icon'><img.*?><\/span>/g, '');
+
+  // 转发的微博
+  if (status.retweeted_status) {
+    tempHTML += "<br><br>";
+    // 可能有转发的微博被删除的情况
+    if (status.retweeted_status.user) {
+      tempHTML += '<div style="border-left: 3px solid gray; padding-left: 1em;">' +
+        '转发 <a href="https://weibo.com/' + status.retweeted_status.user.id + '" target="_blank">@' + status.retweeted_status.user.screen_name + '</a>: ' +
+        statusToHTML(status.retweeted_status) +
+        '</div>';
+    }
+  }
+
+  // 微博配图
+  if (status.pics) {
+    status.pics.forEach(function (item) {
+      tempHTML += "<br><br>";
+      const url = config.imageCache ? (config.imageCache + encodeURIComponent(item.large.url)) : item.large.url;
+      const largeUrl = config.imageCache ? (config.imageCache + encodeURIComponent(item.large.url)) : item.large.url;
+      tempHTML += '<a href="' + largeUrl + '" target="_blank"><img src="' + url+ '"></a>';
+    });
+  }
+
+  return tempHTML;
+}

src/types.ts

@@ -0,0 +1,81 @@
+import { Tracer } from "tracer";
+import { WeiboData } from "./modules/weibo/weibo";
+
+export interface RSSKoaContext {
+  cache: CacheInterface;
+  weibo: WeiboData;
+}
+
+export interface RSSKoaState {
+  // cache hit
+  hit: 0 | 1;
+}
+
+export type LoggerInterface = Tracer.Logger<string>;
+
+export interface CacheInterface {
+  set: (key: string, value: any, expire: number) => Promise<void>;
+  get: (key: string) => any;
+  memo: <T>(cb: () => T, key: string, expire: number) => Promise<Awaited<T>>;
+}
+
+export interface WeiboStatus {
+  // eg: '4851725594528288'
+  id: string;
+  // eg: '4851725594528288'
+  mid: string;
+  // eg: 'MlHCnj00U'
+  bid: string;
+  // time
+  created_at: string;
+  // text
+  text: string;
+  isLongText: boolean;
+  user: {
+    id: number,
+    screen_name: string,
+    profile_url: string;
+    description: string;
+    // more...
+    [x: string]: any;
+  },
+  // pics
+  pic_ids: string[];
+  thumbnail_pic: string;
+  bmiddle_pic: string;
+  original_pic: string;
+  pics: {
+    pid: string,
+    url: string,
+    size: 'orj360' | 'large',
+    large: {
+      size: 'orj360' | 'large',
+      url: string,
+    }
+  }[];
+  // video and other data
+  page_info?: {
+    type: 'video' | 'search_topic';
+    // eg: 'http://t.cn/A6K3ITkN'
+    url_ori: string;
+    // eg: '搜狐新闻的微博视频'
+    page_title: string;
+    // eg: '2022触动瞬间'
+    title: string;
+    // eg: '搜狐新闻的微博视频'
+    content1: string;
+    content2: string;
+    // more...
+    [x: string]: any;
+  };
+  retweeted_status?: WeiboStatus;
+  [x: string]: any;
+}
+
+export interface WeiboUserData {
+  uid: string,
+  screenName: string,
+  description: string,
+  containerId?: string,
+  statusList?: WeiboStatus[],
+}

src/utils.ts

@@ -0,0 +1,19 @@
+
+export const normalizePort = (val: string) => {
+  const port = parseInt(val, 10);
+  if (isNaN(port)) {
+    // named pipe
+    return val;
+  }
+  if (port >= 0) {
+    // port number
+    return port;
+  }
+  return false;
+}
+
+export const waitMs = (ms: number) => {
+  return new Promise(resolve => {
+    setTimeout(resolve, ms);
+  });
+};