From 656ee0bcb4e7752962dbf2b36caa8da8ca5616ef Mon Sep 17 00:00:00 2001 From: anian Date: Tue, 21 Apr 2026 15:56:25 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8E=A7=E5=88=B6=E5=8F=B0?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E9=A1=B9=EF=BC=9B=E4=BF=AE=E6=94=B9=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E5=88=97=E8=A1=A8=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 + gradle.properties | 2 +- .../la/moony/friends/RssFeedSyncEvent.java | 14 +- .../friends/extension/CronFriendPost.java | 5 + .../moony/friends/extension/FriendPost.java | 3 + .../friends/reconciler/RssSyncReconciler.java | 139 ++++++++++++++---- .../impl/RssFeedSyncLogServiceImpl.java | 17 ++- ui/src/api/generated/models/cron-spec.ts | 30 ++++ .../api/generated/models/friend-post-spec.ts | 6 + ui/src/views/CronTab.vue | 37 +++++ 10 files changed, 226 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 8cf448c..1a9e653 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ > 基于原作者v1.4.3代码修改 +#### v1.4.3-4 +1. 控制台新增并行处理数量、图片下载开关、图片下载位置、图片下载代理等配置 +2. 修改图片列表字段为imageFiles（不再覆盖title） + #### v1.4.3-3 1. title字段改为微博图片列表 2. 图片列表内文件将自动下载至/upload/image-host diff --git a/gradle.properties b/gradle.properties index 4310a01..8f8d2a6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=1.4.3-3 +version=1.4.3-4 diff --git a/src/main/java/la/moony/friends/RssFeedSyncEvent.java b/src/main/java/la/moony/friends/RssFeedSyncEvent.java index fb7d716..4e5cd6e 100644 --- a/src/main/java/la/moony/friends/RssFeedSyncEvent.java +++ b/src/main/java/la/moony/friends/RssFeedSyncEvent.java @@ -9,12 +9,24 @@ import java.util.List; public class RssFeedSyncEvent extends ApplicationEvent { private final Link link; private final int sum; + private final int parallelCount; + private final boolean enableWeiboImageRecordAndDownload; private final List disableSyncList; + private final String imageSavePath; + private final String imageBaseUrl; + private final String imageReferer; - public RssFeedSyncEvent(Object source, Link link, int sum, List disableSyncList) { + public RssFeedSyncEvent(Object source, Link link, int sum, int parallelCount, + boolean enableWeiboImageRecordAndDownload, List disableSyncList, + String imageSavePath, String imageBaseUrl, String imageReferer) { super(source); this.link = link; this.sum = sum; + this.parallelCount = parallelCount; + this.enableWeiboImageRecordAndDownload = enableWeiboImageRecordAndDownload; this.disableSyncList = disableSyncList; + this.imageSavePath = imageSavePath; + this.imageBaseUrl = imageBaseUrl; + this.imageReferer = imageReferer; } } diff --git a/src/main/java/la/moony/friends/extension/CronFriendPost.java b/src/main/java/la/moony/friends/extension/CronFriendPost.java index 621f745..c99d40d 100644 --- a/src/main/java/la/moony/friends/extension/CronFriendPost.java +++ b/src/main/java/la/moony/friends/extension/CronFriendPost.java @@ -33,6 +33,11 @@ public class CronFriendPost extends AbstractExtension { private String cron; private String timezone; private boolean suspend; + private boolean enableWeiboImageRecordAndDownload; + private int parallelCount; + private String imageSavePath; + private String imageBaseUrl; + private String imageReferer; @Schema( minimum = "0" diff --git a/src/main/java/la/moony/friends/extension/FriendPost.java b/src/main/java/la/moony/friends/extension/FriendPost.java index a8f4800..bab1f10 100644 --- a/src/main/java/la/moony/friends/extension/FriendPost.java +++ b/src/main/java/la/moony/friends/extension/FriendPost.java @@ -38,6 +38,9 @@ public class FriendPost extends AbstractExtension { private Instant pubDate; private String linkName; + + // Comma-separated image file names extracted from content. + private String imageFiles; } } diff --git a/src/main/java/la/moony/friends/reconciler/RssSyncReconciler.java b/src/main/java/la/moony/friends/reconciler/RssSyncReconciler.java index fd418a5..3560292 100644 --- a/src/main/java/la/moony/friends/reconciler/RssSyncReconciler.java +++ b/src/main/java/la/moony/friends/reconciler/RssSyncReconciler.java @@ -7,7 +7,6 @@ import com.rometools.rome.feed.synd.SyndFeed; import com.rometools.rome.io.SyndFeedInput; import com.rometools.rome.io.XmlReader; import la.moony.friends.RssFeedSyncEvent; -import la.moony.friends.extension.CronFriendPost; import la.moony.friends.extension.FriendPost; import la.moony.friends.extension.Link; import la.moony.friends.extension.RssFeedSyncLog; @@ -36,6 +35,7 @@ import run.halo.app.extension.controller.RequestQueue; import run.halo.app.extension.router.selector.FieldSelector; import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -49,6 +49,7 @@ import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.Semaphore; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -74,8 +75,9 @@ public class RssSyncReconciler implements Reconciler, Pattern.CASE_INSENSITIVE ); - private static final String IMAGE_CDN_BASE_URL = "https://cdn.img2ipfs.com/weibo/middle/"; - private static final Path IMAGE_HOST_DIR = Path.of("/root/.halo2/attachments/upload/image-host/"); + private static final String DEFAULT_IMAGE_BASE_URL = "https://wx3.sinaimg.cn/middle/"; + private static final String DEFAULT_IMAGE_REFERER = "https://weibo.com/"; + private static final String DEFAULT_IMAGE_HOST_PATH = "/root/.halo2/attachments/upload/image-host/"; private volatile boolean running = false; @@ -83,6 +85,8 @@ public class RssSyncReconciler implements Reconciler, private final RequestQueue queue; private final Controller controller; private final ObjectMapper objectMapper = Unstructured.OBJECT_MAPPER; + private volatile int configuredParallelCount = 1; + private volatile Semaphore syncSemaphore = new Semaphore(1, true); public RssSyncReconciler(ExtensionClient client) { this.client = client; @@ -92,9 +96,22 @@ public class RssSyncReconciler implements Reconciler, @Override public Result reconcile(Request request) { + int parallelCount = normalizeParallelCount(request.parallelCount()); + refreshSyncSemaphore(parallelCount); + Semaphore semaphore = this.syncSemaphore; + boolean acquired = false; + try { + semaphore.acquire(); + acquired = true; + Link link = request.link(); int sum = request.sum(); List disableSyncList = request.disableSyncList(); + boolean enableWeiboImageRecordAndDownload = + request.enableWeiboImageRecordAndDownload(); + String imageSavePath = request.imageSavePath(); + String imageBaseUrl = request.imageBaseUrl(); + String imageReferer = request.imageReferer(); var linkName = link.getMetadata().getName(); String rssUrl = getRss(link); boolean isContains = false; @@ -108,7 +125,8 @@ public class RssSyncReconciler implements Reconciler, newRssFeedSyncLog.setMetadata(metadata); newRssFeedSyncLog.setLinkName(linkName); if (StringUtils.isNotEmpty(rssUrl) && !isContains) { - tryToSynchronizeFriendPost(newRssFeedSyncLog, link, sum); + tryToSynchronizeFriendPost(newRssFeedSyncLog, link, sum, imageSavePath, + imageBaseUrl, imageReferer, enableWeiboImageRecordAndDownload); } else if (StringUtils.isEmpty(rssUrl)) { newRssFeedSyncLog.setSyncTime(Instant.now()); newRssFeedSyncLog.setState(RssFeedSyncLog.RssFeedSyncLogState.nolink); @@ -132,6 +150,27 @@ public class RssSyncReconciler implements Reconciler, client.create(newRssFeedSyncLog); } return Result.doNotRetry(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.warn("Interrupted while waiting for sync permit.", e); + return Result.doNotRetry(); + } finally { + if (acquired) { + semaphore.release(); + } + } + } + + private synchronized void refreshSyncSemaphore(int parallelCount) { + if (parallelCount == configuredParallelCount) { + return; + } + configuredParallelCount = parallelCount; + syncSemaphore = new Semaphore(parallelCount, true); + } + + private int normalizeParallelCount(int parallelCount) { + return parallelCount <= 0 ? 1 : parallelCount; } public String getRss(Link link) { @@ -166,15 +205,12 @@ public class RssSyncReconciler implements Reconciler, client.update(extension); } - private void tryToSynchronizeFriendPost(RssFeedSyncLog syncLog, Link link, int sum) { + private void tryToSynchronizeFriendPost(RssFeedSyncLog syncLog, Link link, int sum, + String imageSavePath, String imageBaseUrl, String imageReferer, + boolean enableWeiboImageRecordAndDownload) { var linkName = link.getMetadata().getName(); var annotations = nullSafeAnnotations(link); var rssUrl = annotations.get("rss_url"); - - CronFriendPost cron = new CronFriendPost(); - var spec = new CronFriendPost.CronSpec(); - spec.setSuccessfulRetainLimit(5); - cron.setSpec(spec); syncLog.setSyncTime(Instant.now()); List friendPostList = fetchFriendPost(rssUrl, sum, syncLog); @@ -214,11 +250,12 @@ public class RssSyncReconciler implements Reconciler, } } - if (!imageFileNames.isEmpty()) { - String imageTitle = "img:::" + String.join(",", imageFileNames); - friendPostSpec.setTitle(imageTitle); + if (enableWeiboImageRecordAndDownload && !imageFileNames.isEmpty()) { + String imageFiles = String.join(",", imageFileNames); + friendPostSpec.setImageFiles(imageFiles); // 通知图片处理服务 - notifyImageService(imageTitle); + notifyImageService(imageFiles, imageSavePath, imageBaseUrl, + imageReferer); } String pureText = CommonUtils.parseAndTruncateHtml2Text(description, 100000); @@ -300,22 +337,28 @@ public class RssSyncReconciler implements Reconciler, /** * 通知图片处理服务有新的图片需要处理 - * @param imageTitle 图片标题，格式为 "img:::file1,file2,..." + * @param imageFiles 图片文件名，英文逗号分隔，如 "file1,file2,..." */ - private void notifyImageService(String imageTitle) { - if (StringUtils.isEmpty(imageTitle)) { + private void notifyImageService(String imageFiles, String imageSavePath, String imageBaseUrl, + String imageReferer) { + if (StringUtils.isBlank(imageFiles)) { return; } - new Thread(() -> downloadImagesByTitle(imageTitle)).start(); + new Thread(() -> downloadImagesByFiles(imageFiles, imageSavePath, imageBaseUrl, + imageReferer)).start(); } - private void downloadImagesByTitle(String imageTitle) { - if (!imageTitle.startsWith("img:::")) { + private void downloadImagesByFiles(String imageFiles, String imageSavePath, + String imageBaseUrl, String imageReferer) { + if (StringUtils.isBlank(imageFiles)) { return; } try { - Files.createDirectories(IMAGE_HOST_DIR); - String[] fileNames = imageTitle.substring("img:::".length()).split(","); + Path imageHostDir = resolveImageHostDir(imageSavePath); + String resolvedBaseUrl = resolveImageBaseUrl(imageBaseUrl); + String resolvedReferer = resolveImageReferer(imageBaseUrl, imageReferer); + Files.createDirectories(imageHostDir); + String[] fileNames = imageFiles.split(","); for (String fileName : fileNames) { String trimmedName = fileName.trim(); if (StringUtils.isEmpty(trimmedName)) { @@ -324,18 +367,52 @@ public class RssSyncReconciler implements Reconciler, String encodedFileName = URLEncoder.encode(trimmedName, StandardCharsets.UTF_8) .replace("+", "%20"); - String downloadUrl = IMAGE_CDN_BASE_URL + encodedFileName; - Path targetPath = IMAGE_HOST_DIR.resolve(trimmedName); + String downloadUrl = resolvedBaseUrl + encodedFileName; + Path targetPath = imageHostDir.resolve(trimmedName); - try (InputStream inputStream = new URL(downloadUrl).openStream()) { + HttpURLConnection connection = (HttpURLConnection) new URL(downloadUrl) + .openConnection(); + if (StringUtils.isNotBlank(resolvedReferer)) { + connection.setRequestProperty("Referer", resolvedReferer); + } + try (InputStream inputStream = connection.getInputStream()) { Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING); + } finally { + connection.disconnect(); } } } catch (Exception e) { - log.warn("Failed to download images for title: {}", imageTitle, e); + log.warn("Failed to download images for files: {}", imageFiles, e); } } + private Path resolveImageHostDir(String imageSavePath) { + if (StringUtils.isBlank(imageSavePath)) { + return Path.of(DEFAULT_IMAGE_HOST_PATH); + } + return Path.of(imageSavePath.trim()); + } + + private String resolveImageBaseUrl(String imageBaseUrl) { + String resolvedBaseUrl = StringUtils.isBlank(imageBaseUrl) + ? DEFAULT_IMAGE_BASE_URL + : imageBaseUrl.trim(); + if (!resolvedBaseUrl.endsWith("/")) { + return resolvedBaseUrl + "/"; + } + return resolvedBaseUrl; + } + + private String resolveImageReferer(String imageBaseUrl, String imageReferer) { + if (StringUtils.isBlank(imageBaseUrl)) { + return DEFAULT_IMAGE_REFERER; + } + if (StringUtils.isBlank(imageReferer)) { + return null; + } + return imageReferer.trim(); + } + static int pageNullSafe(Integer page) { return ObjectUtils.defaultIfNull(page, 1); } @@ -375,10 +452,16 @@ public class RssSyncReconciler implements Reconciler, @EventListener(RssFeedSyncEvent.class) public void onReplyEvent(RssFeedSyncEvent event) { - var request = new Request(event.getLink(), event.getSum(), event.getDisableSyncList()); + var request = new Request(event.getLink(), event.getSum(), event.getDisableSyncList(), + event.isEnableWeiboImageRecordAndDownload(), + event.getImageSavePath(), event.getImageBaseUrl(), event.getImageReferer(), + event.getParallelCount()); queue.addImmediately(request); } - public record Request(Link link, int sum, List disableSyncList) { + public record Request(Link link, int sum, List disableSyncList, + boolean enableWeiboImageRecordAndDownload, + String imageSavePath, String imageBaseUrl, + String imageReferer, int parallelCount) { } } \ No newline at end of file diff --git a/src/main/java/la/moony/friends/service/impl/RssFeedSyncLogServiceImpl.java b/src/main/java/la/moony/friends/service/impl/RssFeedSyncLogServiceImpl.java index 07a7dc5..bd9b952 100644 --- a/src/main/java/la/moony/friends/service/impl/RssFeedSyncLogServiceImpl.java +++ b/src/main/java/la/moony/friends/service/impl/RssFeedSyncLogServiceImpl.java @@ -74,14 +74,25 @@ public class RssFeedSyncLogServiceImpl implements RssFeedSyncLogService { CronFriendPost cron = new CronFriendPost(); CronFriendPost.CronSpec cronSpec = new CronFriendPost.CronSpec(); cronSpec.setSuccessfulRetainLimit(5); + cronSpec.setParallelCount(1); + cronSpec.setEnableWeiboImageRecordAndDownload(false); cronSpec.setDisableSyncList(new ArrayList<>()); + cronSpec.setImageSavePath(""); + cronSpec.setImageBaseUrl(""); + cronSpec.setImageReferer(""); cron.setSpec(cronSpec); return client.fetch(CronFriendPost.class, "cron-friends-default") .defaultIfEmpty(cron) .flatMap(cronFriendPost -> { var spec = cronFriendPost.getSpec(); int successfulRetainLimit = spec.getSuccessfulRetainLimit(); + int parallelCount = spec.getParallelCount(); + boolean enableWeiboImageRecordAndDownload = + spec.isEnableWeiboImageRecordAndDownload(); List disableSyncList = spec.getDisableSyncList(); + String imageSavePath = spec.getImageSavePath(); + String imageBaseUrl = spec.getImageBaseUrl(); + String imageReferer = spec.getImageReferer(); int sum = successfulRetainLimit == 0 ? 5 : successfulRetainLimit; var listOptions = new ListOptions(); FieldSelector fieldSelector = FieldSelector.of(isNull("metadata.deletionTimestamp")); @@ -90,7 +101,11 @@ public class RssFeedSyncLogServiceImpl implements RssFeedSyncLogService { } listOptions.setFieldSelector(fieldSelector); return client.listAll(Link.class, listOptions, Sort.by("metadata.creationTimestamp")) - .doOnNext(link -> eventPublisher.publishEvent(new RssFeedSyncEvent(this, link,sum,disableSyncList))) + .doOnNext(link -> eventPublisher.publishEvent( + new RssFeedSyncEvent(this, link, sum, parallelCount, + enableWeiboImageRecordAndDownload, disableSyncList, imageSavePath, + imageBaseUrl, imageReferer) + )) .then(); }); } diff --git a/ui/src/api/generated/models/cron-spec.ts b/ui/src/api/generated/models/cron-spec.ts index 1f230c1..cf2f77e 100644 --- a/ui/src/api/generated/models/cron-spec.ts +++ b/ui/src/api/generated/models/cron-spec.ts @@ -32,6 +32,36 @@ export interface CronSpec { * @memberof CronSpec */ 'disableSyncList'?: Array; + /** + * + * @type {number} + * @memberof CronSpec + */ + 'parallelCount'?: number; + /** + * + * @type {boolean} + * @memberof CronSpec + */ + 'enableWeiboImageRecordAndDownload'?: boolean; + /** + * + * @type {string} + * @memberof CronSpec + */ + 'imageSavePath'?: string; + /** + * + * @type {string} + * @memberof CronSpec + */ + 'imageBaseUrl'?: string; + /** + * + * @type {string} + * @memberof CronSpec + */ + 'imageReferer'?: string; /** * * @type {number} diff --git a/ui/src/api/generated/models/friend-post-spec.ts b/ui/src/api/generated/models/friend-post-spec.ts index 3ecafca..afed120 100644 --- a/ui/src/api/generated/models/friend-post-spec.ts +++ b/ui/src/api/generated/models/friend-post-spec.ts @@ -38,6 +38,12 @@ export interface FriendPostSpec { * @memberof FriendPostSpec */ 'description'?: string; + /** + * Comma-separated image file names. + * @type {string} + * @memberof FriendPostSpec + */ + 'imageFiles'?: string; /** * * @type {string} diff --git a/ui/src/views/CronTab.vue b/ui/src/views/CronTab.vue index 1a35cbd..2f0d316 100644 --- a/ui/src/views/CronTab.vue +++ b/ui/src/views/CronTab.vue @@ -18,7 +18,12 @@ const initialFormState: CronFriendPost = { cron: "@daily", timezone:"Asia/Shanghai", suspend: false, + enableWeiboImageRecordAndDownload: false, + parallelCount: 1, successfulRetainLimit: 0, + imageSavePath: "", + imageBaseUrl: "", + imageReferer: "", disableSyncList: undefined, }, kind: "CronFriendPost", @@ -130,6 +135,20 @@ const { mutate:save, isLoading:saveIsLoading } = useMutation({ }, ]" /> + + + + +