diff --git a/README.md b/README.md index 330a499..8cf448c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ > 基于原作者v1.4.3代码修改 +#### v1.4.3-3 +1. title字段改为微博图片列表 +2. 图片列表内文件将自动下载至/upload/image-host + #### v1.4.3-2 1. 新增URL参数author用于查询作者 2. 模板参数新增${authors}返回所有作者 diff --git a/gradle.properties b/gradle.properties index fbde1f1..4310a01 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=1.4.3-2 +version=1.4.3-3 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5a72425..dc419c8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -87,7 +87,7 @@ importers: specifier: ^2.21.0 version: 2.22.0(@tiptap/extension-collaboration@3.20.0(@tiptap/core@3.20.0(@tiptap/pm@3.20.0))(@tiptap/pm@3.20.0)(@tiptap/y-tiptap@3.0.2(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(y-protocols@1.0.7(yjs@13.6.29))(yjs@13.6.29))(yjs@13.6.29))(@tiptap/extension-node-range@3.20.0(@tiptap/core@3.20.0(@tiptap/pm@3.20.0))(@tiptap/pm@3.20.0))(@tiptap/y-tiptap@3.0.2(prosemirror-model@1.25.4)(prosemirror-state@1.4.4)(prosemirror-view@1.41.6)(y-protocols@1.0.7(yjs@13.6.29))(yjs@13.6.29))(axios@1.13.6)(vue-router@4.6.4(vue@3.5.29(typescript@5.5.4)))(vue@3.5.29(typescript@5.5.4)) '@kunkunyu/fridends-rss': - specifier: file:../packages/fridends-rss + specifier: workspace:* version: link:../packages/fridends-rss '@tanstack/vue-query': specifier: ^4.26.1 @@ -917,48 +917,56 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [glibc] '@oxc-parser/binding-linux-arm64-musl@0.115.0': resolution: {integrity: sha512-HjsZbJPH9mMd4swJRywVMsDZsJX0hyKb1iNHo5ijRl5yhtbO3lj7ImSrrL1oZ1VEg0te4iKmDGGz/6YPLd1G8w==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [musl] '@oxc-parser/binding-linux-ppc64-gnu@0.115.0': resolution: {integrity: sha512-zhhePoBrd7kQx3oClX/W6NldsuCbuMqaN9rRsY+6/WoorAb4j490PG/FjqgAXscWp2uSW2WV9L+ksn0wHrvsrg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@oxc-parser/binding-linux-riscv64-gnu@0.115.0': resolution: {integrity: sha512-t/IRojvUE9XrKu+/H1b8YINug+7Q6FLls5rsm2lxB5mnS8GN/eYAYrPgHkcg9/1SueRDSzGpDYu3lGWTObk1zw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [riscv64] os: [linux] + libc: [glibc] '@oxc-parser/binding-linux-riscv64-musl@0.115.0': resolution: {integrity: sha512-79jBHSSh/YpQRAmvYoaCfpyToRbJ/HBrdB7hxK2ku2JMehjopTVo+xMJss/RV7/ZYqeezgjvKDQzapJbgcjVZA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [riscv64] os: [linux] + libc: [musl] '@oxc-parser/binding-linux-s390x-gnu@0.115.0': resolution: {integrity: sha512-nA1TpxkhNTIOMMyiSSsa7XIVJVoOU/SsVrHIz3gHvWweB5PHCQfO7w+Lb2EP0lBWokv7HtA/KbF7aLDoXzmuMw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] + libc: [glibc] '@oxc-parser/binding-linux-x64-gnu@0.115.0': resolution: {integrity: sha512-9iVX789DoC3SaOOG+X6NcF/tVChgLp2vcHffzOC2/Z1JTPlz6bMG2ogvcW6/9s0BG2qvhNQImd+gbWYeQbOwVw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [glibc] '@oxc-parser/binding-linux-x64-musl@0.115.0': resolution: {integrity: sha512-RmQmk+mjCB0nMNfEYhaCxwofLo1Z95ebHw1AGvRiWGCd4zhCNOyskgCbMogIcQzSB3SuEKWgkssyaiQYVAA4hQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [musl] '@oxc-parser/binding-openharmony-arm64@0.115.0': resolution: {integrity: sha512-viigraWWQhhDvX5aGq+wrQq58k00Xq3MHz/0R4AFMxGlZ8ogNonpEfNc73Q5Ly87Z6sU9BvxEdG0dnYTfVnmew==} @@ -1055,66 +1063,79 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} @@ -1180,41 +1201,49 @@ packages: resolution: {integrity: sha512-tzLHo5upqlDWK3wSTit0m0iZ8N6pm6S42R/sfeOcPwERcTjhTrbQ6GOEbmwsay845EgzJbGWwaOzVeGLT55YCw==} cpu: [arm64] os: [linux] + libc: [glibc] '@rspack/binding-linux-arm64-gnu@1.7.7': resolution: {integrity: sha512-uSq4qkvmAzSDUTKE2v4yUgHIBdTily1k3BcK5wBCGFm9OPODj5lQZpAdOHHIwu+Jxyjoa7Mb64tghhj9hZcXcA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rspack/binding-linux-arm64-musl@1.4.0-beta.0': resolution: {integrity: sha512-g3YZCNB+oR8+CG83iOoACZrxiM9sKlB33QmJB2PFk5TTryhkNGEq3vwiyqe7AVPWYfuCCqoRdzPNzWBIN80cEg==} cpu: [arm64] os: [linux] + libc: [musl] '@rspack/binding-linux-arm64-musl@1.7.7': resolution: {integrity: sha512-NhWCBfiu6plpmLRP6c6D5lBUaVrBr1nvjSEc7VyQF8TGh8URo2btH0wngEiX0nWvidsSlERt1l6Y5QPGuiCl1g==} cpu: [arm64] os: [linux] + libc: [musl] '@rspack/binding-linux-x64-gnu@1.4.0-beta.0': resolution: {integrity: sha512-TNIm9APDmcbbrwWgSxaIbq73r0cKrzyS0Ei7rB0TyX9EmFYSfsCdmdJMwG2yKP3p+egRIDMWU9AIrxL4HIMrBg==} cpu: [x64] os: [linux] + libc: [glibc] '@rspack/binding-linux-x64-gnu@1.7.7': resolution: {integrity: sha512-aRvf8gCI7jDeEN9i4u9fY5coa3ZAyHzGVA4ZhTJCgZ5wWA5A9SQewMSq7khS1WAAFE1USlk1tUuPujnrGoYrGg==} cpu: [x64] os: [linux] + libc: [glibc] '@rspack/binding-linux-x64-musl@1.4.0-beta.0': resolution: {integrity: sha512-PCfGShh6y0CqUX8XxuxkEKOBLELuxDZ/sacM047CBIet3CgvqmT0Ff2DKXFIu8Q+NWrKnzvopO7hPv4Zelku6A==} cpu: [x64] os: [linux] + libc: [musl] '@rspack/binding-linux-x64-musl@1.7.7': resolution: {integrity: sha512-ALPto4OT7snzXbYDyqkLfh1BvwDTTH1hPYXGUXBzQ0wEV7sXeyvxCC4yjH6B5MhR7W3tFuF4IfDy5Z4BxmOoGQ==} cpu: [x64] os: [linux] + libc: [musl] '@rspack/binding-wasm32-wasi@1.7.7': resolution: {integrity: sha512-7DZvUp0v75n451qfZw1ppbPakL6NAc2gjb5e9AJcOb7KUMBHNyOxqpPo/jRYKxH7isPpLfpoId79WQGGNTTMAw==} diff --git a/src/main/java/la/moony/friends/reconciler/RssSyncReconciler.java b/src/main/java/la/moony/friends/reconciler/RssSyncReconciler.java index 1cecd01..fd418a5 100644 --- a/src/main/java/la/moony/friends/reconciler/RssSyncReconciler.java +++ b/src/main/java/la/moony/friends/reconciler/RssSyncReconciler.java @@ -27,16 +27,30 @@ import run.halo.app.extension.ListResult; import run.halo.app.extension.Metadata; import run.halo.app.extension.PageRequestImpl; import run.halo.app.extension.Unstructured; -import run.halo.app.extension.controller.*; +import run.halo.app.extension.controller.Controller; +import run.halo.app.extension.controller.ControllerBuilder; +import run.halo.app.extension.controller.DefaultController; +import run.halo.app.extension.controller.DefaultQueue; +import run.halo.app.extension.controller.Reconciler; +import run.halo.app.extension.controller.RequestQueue; import run.halo.app.extension.router.selector.FieldSelector; + +import java.io.InputStream; import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.springframework.data.domain.Sort.Order.asc; import static run.halo.app.extension.MetadataUtil.nullSafeAnnotations; @@ -45,10 +59,24 @@ import static run.halo.app.extension.index.query.QueryFactory.equal; import static run.halo.app.extension.index.query.QueryFactory.isNull; @Component -public class RssSyncReconciler implements Reconciler,SmartLifecycle { +public class RssSyncReconciler implements Reconciler, SmartLifecycle { private static final Logger log = LoggerFactory.getLogger(RssSyncReconciler.class); + // 新增：正则匹配所有img标签的src属性 + private static final Pattern IMG_SRC_PATTERN = Pattern.compile( + "]*?src\\s*=\\s*(['\"])(.*?)\\1[^>]*?>", + Pattern.CASE_INSENSITIVE + ); + + private static final Pattern MW2000_IMAGE_FILE_PATTERN = Pattern.compile( + ".*/mw2000/([^/?#]+)(?:[?#].*)?$", + Pattern.CASE_INSENSITIVE + ); + + private static final String IMAGE_CDN_BASE_URL = "https://cdn.img2ipfs.com/weibo/middle/"; + private static final Path IMAGE_HOST_DIR = Path.of("/root/.halo2/attachments/upload/image-host/"); + private volatile boolean running = false; private final ExtensionClient client; @@ -56,7 +84,6 @@ public class RssSyncReconciler implements Reconciler, private final Controller controller; private final ObjectMapper objectMapper = Unstructured.OBJECT_MAPPER; - public RssSyncReconciler(ExtensionClient client) { this.client = client; queue = new DefaultQueue<>(Instant::now); @@ -82,12 +109,12 @@ public class RssSyncReconciler implements Reconciler, newRssFeedSyncLog.setLinkName(linkName); if (StringUtils.isNotEmpty(rssUrl) && !isContains) { tryToSynchronizeFriendPost(newRssFeedSyncLog, link, sum); - }else if (StringUtils.isEmpty(rssUrl)) { + } else if (StringUtils.isEmpty(rssUrl)) { newRssFeedSyncLog.setSyncTime(Instant.now()); newRssFeedSyncLog.setState(RssFeedSyncLog.RssFeedSyncLogState.nolink); newRssFeedSyncLog.setFailureReason("No RSS link"); newRssFeedSyncLog.setFailureMessage("无 RSS 链接."); - }else if (isContains) { + } else if (isContains) { newRssFeedSyncLog.setSyncTime(Instant.now()); newRssFeedSyncLog.setState(RssFeedSyncLog.RssFeedSyncLogState.failed); newRssFeedSyncLog.setFailureReason("admin disables sync"); @@ -101,7 +128,7 @@ public class RssSyncReconciler implements Reconciler, rssFeedSyncLog.setFailureReason(newRssFeedSyncLog.getFailureReason()); rssFeedSyncLog.setFailureMessage(newRssFeedSyncLog.getFailureMessage()); client.update(rssFeedSyncLog); - }else { + } else { client.create(newRssFeedSyncLog); } return Result.doNotRetry(); @@ -121,10 +148,10 @@ public class RssSyncReconciler implements Reconciler, linkRss = rssUri; annotations.remove("rss_uri"); annotations.put("rss_url", rssUri); - }else { - annotations.put("is_request","true"); + } else { + annotations.put("is_request", "true"); if (StringUtils.isNotEmpty(linkRss)) { - annotations.put("rss_url",linkRss); + annotations.put("rss_url", linkRss); } } updateLink(link); @@ -133,7 +160,6 @@ public class RssSyncReconciler implements Reconciler, return rssUrl; } - public void updateLink(Link link) { Map extensionMap = objectMapper.convertValue(link, Map.class); var extension = new Unstructured(extensionMap); @@ -151,19 +177,19 @@ public class RssSyncReconciler implements Reconciler, cron.setSpec(spec); syncLog.setSyncTime(Instant.now()); List friendPostList = fetchFriendPost(rssUrl, sum, syncLog); - + // 即使获取失败也继续处理 if (friendPostList != null && !friendPostList.isEmpty()) { for (FriendPost rssPost : friendPostList) { var friendPostListOptions = new ListOptions(); friendPostListOptions.setFieldSelector(FieldSelector.of( and(isNull("metadata.deletionTimestamp"), - equal("spec.postLink",rssPost.getSpec().getPostLink()) + equal("spec.postLink", rssPost.getSpec().getPostLink()) ) )); long total = client.listBy(FriendPost.class, friendPostListOptions, PageRequestImpl.ofSize(1)).getTotal(); - if (total==0) { + if (total == 0) { FriendPost friendPost = new FriendPost(); // 设置元数据才能保存 friendPost.setMetadata(new Metadata()); @@ -175,23 +201,42 @@ public class RssSyncReconciler implements Reconciler, friendPostSpec.setAuthorUrl(link.getSpec().getUrl()); friendPostSpec.setLinkName(linkName); String description = friendPost.getSpec().getDescription(); - //解析html内容转换成文本 - if (StringUtils.isNotEmpty(description)){ - description = CommonUtils.parseAndTruncateHtml2Text(description, 100000); + + // 先提取图片文件名覆盖标题，再将 HTML 描述转换为纯文本 + if (StringUtils.isNotEmpty(description)) { + List imageFileNames = new ArrayList<>(); + Matcher matcher = IMG_SRC_PATTERN.matcher(description); + while (matcher.find()) { + String imgSrc = matcher.group(2); + Matcher mw2000Matcher = MW2000_IMAGE_FILE_PATTERN.matcher(imgSrc); + if (mw2000Matcher.matches()) { + imageFileNames.add(mw2000Matcher.group(1)); + } + } + + if (!imageFileNames.isEmpty()) { + String imageTitle = "img:::" + String.join(",", imageFileNames); + friendPostSpec.setTitle(imageTitle); + // 通知图片处理服务 + notifyImageService(imageTitle); + } + + String pureText = CommonUtils.parseAndTruncateHtml2Text(description, 100000); String regexp = "[　*|\\s*]*"; - description = description.replaceFirst(regexp, "").trim(); + pureText = pureText.replaceFirst(regexp, "").trim(); String weiboVideoRegex = "\\s[^\\s]*的微博视频 视频无法显示，请前往微博视频观看。"; - description = description.replaceAll(weiboVideoRegex, ""); - friendPostSpec.setDescription(description); + pureText = pureText.replaceAll(weiboVideoRegex, ""); + friendPostSpec.setDescription(pureText); } + client.create(friendPost); } } - delFriendPost(linkName,sum); + delFriendPost(linkName, sum); } } - public List fetchFriendPost(String rssAddress,int postsLimit, RssFeedSyncLog sync) { + public List fetchFriendPost(String rssAddress, int postsLimit, RssFeedSyncLog sync) { List friendPostList = new ArrayList<>(); try { SyndFeed feed = new SyndFeedInput().build(new XmlReader(new URL(rssAddress))); @@ -205,7 +250,7 @@ public class RssSyncReconciler implements Reconciler, SyndContent content = entry.getDescription(); if (content != null) { description = content.getValue(); - }else { + } else { List contents = entry.getContents(); if (!contents.isEmpty()) { description = contents.get(0).getValue(); @@ -235,10 +280,10 @@ public class RssSyncReconciler implements Reconciler, return friendPostList; } - public void delFriendPost(String linkName,Integer sum) { + public void delFriendPost(String linkName, Integer sum) { var friendPostListOptions = new ListOptions(); friendPostListOptions.setFieldSelector(FieldSelector.of( - equal("spec.linkName",linkName) + equal("spec.linkName", linkName) )); long total = client.listBy(FriendPost.class, friendPostListOptions, PageRequestImpl.ofSize(1)).getTotal(); if (total > sum) { @@ -253,6 +298,44 @@ public class RssSyncReconciler implements Reconciler, } } + /** + * 通知图片处理服务有新的图片需要处理 + * @param imageTitle 图片标题，格式为 "img:::file1,file2,..." + */ + private void notifyImageService(String imageTitle) { + if (StringUtils.isEmpty(imageTitle)) { + return; + } + new Thread(() -> downloadImagesByTitle(imageTitle)).start(); + } + + private void downloadImagesByTitle(String imageTitle) { + if (!imageTitle.startsWith("img:::")) { + return; + } + try { + Files.createDirectories(IMAGE_HOST_DIR); + String[] fileNames = imageTitle.substring("img:::".length()).split(","); + for (String fileName : fileNames) { + String trimmedName = fileName.trim(); + if (StringUtils.isEmpty(trimmedName)) { + continue; + } + + String encodedFileName = URLEncoder.encode(trimmedName, StandardCharsets.UTF_8) + .replace("+", "%20"); + String downloadUrl = IMAGE_CDN_BASE_URL + encodedFileName; + Path targetPath = IMAGE_HOST_DIR.resolve(trimmedName); + + try (InputStream inputStream = new URL(downloadUrl).openStream()) { + Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING); + } + } + } catch (Exception e) { + log.warn("Failed to download images for title: {}", imageTitle, e); + } + } + static int pageNullSafe(Integer page) { return ObjectUtils.defaultIfNull(page, 1); } @@ -261,7 +344,6 @@ public class RssSyncReconciler implements Reconciler, return ObjectUtils.defaultIfNull(size, 10); } - @Override public Controller setupWith(ControllerBuilder builder) { return new DefaultController<>( @@ -293,10 +375,10 @@ public class RssSyncReconciler implements Reconciler, @EventListener(RssFeedSyncEvent.class) public void onReplyEvent(RssFeedSyncEvent event) { - var request = new Request(event.getLink(),event.getSum(),event.getDisableSyncList()); + var request = new Request(event.getLink(), event.getSum(), event.getDisableSyncList()); queue.addImmediately(request); } - public record Request(Link link,int sum,List disableSyncList) { + public record Request(Link link, int sum, List disableSyncList) { } -} +} \ No newline at end of file