新增控制台配置项；修改图片列表字段

README.md

@@ -2,6 +2,10 @@
 
 > 基于原作者v1.4.3代码修改
 
+#### v1.4.3-4
+1. 控制台新增并行处理数量、图片下载开关、图片下载位置、图片下载代理等配置
+2. 修改图片列表字段为imageFiles（不再覆盖title）
+
 #### v1.4.3-3
 1. title字段改为微博图片列表
 2. 图片列表内文件将自动下载至/upload/image-host

gradle.properties

@@ -1 +1 @@
-version=1.4.3-3
+version=1.4.3-4

src/main/java/la/moony/friends/RssFeedSyncEvent.java

@@ -9,12 +9,24 @@ import java.util.List;
 public class RssFeedSyncEvent extends ApplicationEvent {
     private final Link link;
     private final int sum;
+    private final int parallelCount;
+    private final boolean enableWeiboImageRecordAndDownload;
     private final List<String> disableSyncList;
+    private final String imageSavePath;
+    private final String imageBaseUrl;
+    private final String imageReferer;
 
-    public RssFeedSyncEvent(Object source, Link link, int sum, List<String> disableSyncList) {
+    public RssFeedSyncEvent(Object source, Link link, int sum, int parallelCount,
+        boolean enableWeiboImageRecordAndDownload, List<String> disableSyncList,
+        String imageSavePath, String imageBaseUrl, String imageReferer) {
         super(source);
         this.link = link;
         this.sum = sum;
+        this.parallelCount = parallelCount;
+        this.enableWeiboImageRecordAndDownload = enableWeiboImageRecordAndDownload;
         this.disableSyncList = disableSyncList;
+        this.imageSavePath = imageSavePath;
+        this.imageBaseUrl = imageBaseUrl;
+        this.imageReferer = imageReferer;
     }
 }

src/main/java/la/moony/friends/extension/CronFriendPost.java

@@ -33,6 +33,11 @@ public class CronFriendPost extends AbstractExtension {
         private String cron;
         private String timezone;
         private boolean suspend;
+        private boolean enableWeiboImageRecordAndDownload;
+        private int parallelCount;
+        private String imageSavePath;
+        private String imageBaseUrl;
+        private String imageReferer;
 
         @Schema(
             minimum = "0"

src/main/java/la/moony/friends/extension/FriendPost.java

@@ -38,6 +38,9 @@ public class FriendPost extends AbstractExtension {
         private Instant pubDate;
 
         private String linkName;
+
+        // Comma-separated image file names extracted from content.
+        private String imageFiles;
     }
 
 }

src/main/java/la/moony/friends/reconciler/RssSyncReconciler.java

@@ -7,7 +7,6 @@ import com.rometools.rome.feed.synd.SyndFeed;
 import com.rometools.rome.io.SyndFeedInput;
 import com.rometools.rome.io.XmlReader;
 import la.moony.friends.RssFeedSyncEvent;
-import la.moony.friends.extension.CronFriendPost;
 import la.moony.friends.extension.FriendPost;
 import la.moony.friends.extension.Link;
 import la.moony.friends.extension.RssFeedSyncLog;
@@ -36,6 +35,7 @@ import run.halo.app.extension.controller.RequestQueue;
 import run.halo.app.extension.router.selector.FieldSelector;
 
 import java.io.InputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
@@ -49,6 +49,7 @@ import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.Semaphore;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -74,8 +75,9 @@ public class RssSyncReconciler implements Reconciler<RssSyncReconciler.Request>,
         Pattern.CASE_INSENSITIVE
     );
 
-    private static final String IMAGE_CDN_BASE_URL = "https://cdn.img2ipfs.com/weibo/middle/";
-    private static final Path IMAGE_HOST_DIR = Path.of("/root/.halo2/attachments/upload/image-host/");
+    private static final String DEFAULT_IMAGE_BASE_URL = "https://wx3.sinaimg.cn/middle/";
+    private static final String DEFAULT_IMAGE_REFERER = "https://weibo.com/";
+    private static final String DEFAULT_IMAGE_HOST_PATH = "/root/.halo2/attachments/upload/image-host/";
 
     private volatile boolean running = false;
 
@@ -83,6 +85,8 @@ public class RssSyncReconciler implements Reconciler<RssSyncReconciler.Request>,
     private final RequestQueue<Request> queue;
     private final Controller controller;
     private final ObjectMapper objectMapper = Unstructured.OBJECT_MAPPER;
+    private volatile int configuredParallelCount = 1;
+    private volatile Semaphore syncSemaphore = new Semaphore(1, true);
 
     public RssSyncReconciler(ExtensionClient client) {
         this.client = client;
@@ -92,9 +96,22 @@ public class RssSyncReconciler implements Reconciler<RssSyncReconciler.Request>,
 
     @Override
     public Result reconcile(Request request) {
+        int parallelCount = normalizeParallelCount(request.parallelCount());
+        refreshSyncSemaphore(parallelCount);
+        Semaphore semaphore = this.syncSemaphore;
+        boolean acquired = false;
+        try {
+            semaphore.acquire();
+            acquired = true;
+
         Link link = request.link();
         int sum = request.sum();
         List<String> disableSyncList = request.disableSyncList();
+        boolean enableWeiboImageRecordAndDownload =
+            request.enableWeiboImageRecordAndDownload();
+        String imageSavePath = request.imageSavePath();
+        String imageBaseUrl = request.imageBaseUrl();
+        String imageReferer = request.imageReferer();
         var linkName = link.getMetadata().getName();
         String rssUrl = getRss(link);
         boolean isContains = false;
@@ -108,7 +125,8 @@ public class RssSyncReconciler implements Reconciler<RssSyncReconciler.Request>,
         newRssFeedSyncLog.setMetadata(metadata);
         newRssFeedSyncLog.setLinkName(linkName);
         if (StringUtils.isNotEmpty(rssUrl) && !isContains) {
-            tryToSynchronizeFriendPost(newRssFeedSyncLog, link, sum);
+            tryToSynchronizeFriendPost(newRssFeedSyncLog, link, sum, imageSavePath,
+                imageBaseUrl, imageReferer, enableWeiboImageRecordAndDownload);
         } else if (StringUtils.isEmpty(rssUrl)) {
             newRssFeedSyncLog.setSyncTime(Instant.now());
             newRssFeedSyncLog.setState(RssFeedSyncLog.RssFeedSyncLogState.nolink);
@@ -132,6 +150,27 @@ public class RssSyncReconciler implements Reconciler<RssSyncReconciler.Request>,
             client.create(newRssFeedSyncLog);
         }
         return Result.doNotRetry();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            log.warn("Interrupted while waiting for sync permit.", e);
+            return Result.doNotRetry();
+        } finally {
+            if (acquired) {
+                semaphore.release();
+            }
+        }
+    }
+
+    private synchronized void refreshSyncSemaphore(int parallelCount) {
+        if (parallelCount == configuredParallelCount) {
+            return;
+        }
+        configuredParallelCount = parallelCount;
+        syncSemaphore = new Semaphore(parallelCount, true);
+    }
+
+    private int normalizeParallelCount(int parallelCount) {
+        return parallelCount <= 0 ? 1 : parallelCount;
     }
 
     public String getRss(Link link) {
@@ -166,15 +205,12 @@ public class RssSyncReconciler implements Reconciler<RssSyncReconciler.Request>,
         client.update(extension);
     }
 
-    private void tryToSynchronizeFriendPost(RssFeedSyncLog syncLog, Link link, int sum) {
+    private void tryToSynchronizeFriendPost(RssFeedSyncLog syncLog, Link link, int sum,
+        String imageSavePath, String imageBaseUrl, String imageReferer,
+        boolean enableWeiboImageRecordAndDownload) {
         var linkName = link.getMetadata().getName();
         var annotations = nullSafeAnnotations(link);
         var rssUrl = annotations.get("rss_url");
-
-        CronFriendPost cron = new CronFriendPost();
-        var spec = new CronFriendPost.CronSpec();
-        spec.setSuccessfulRetainLimit(5);
-        cron.setSpec(spec);
         syncLog.setSyncTime(Instant.now());
         List<FriendPost> friendPostList = fetchFriendPost(rssUrl, sum, syncLog);
 
@@ -214,11 +250,12 @@ public class RssSyncReconciler implements Reconciler<RssSyncReconciler.Request>,
                             }
                         }
 
-                        if (!imageFileNames.isEmpty()) {
-                            String imageTitle = "img:::" + String.join(",", imageFileNames);
-                            friendPostSpec.setTitle(imageTitle);
+                        if (enableWeiboImageRecordAndDownload && !imageFileNames.isEmpty()) {
+                            String imageFiles = String.join(",", imageFileNames);
+                            friendPostSpec.setImageFiles(imageFiles);
                             // 通知图片处理服务
-                            notifyImageService(imageTitle);
+                            notifyImageService(imageFiles, imageSavePath, imageBaseUrl,
+                                imageReferer);
                         }
 
                         String pureText = CommonUtils.parseAndTruncateHtml2Text(description, 100000);
@@ -300,22 +337,28 @@ public class RssSyncReconciler implements Reconciler<RssSyncReconciler.Request>,
 
     /**
      * 通知图片处理服务有新的图片需要处理
-     * @param imageTitle 图片标题，格式为 "img:::file1,file2,..."
+     * @param imageFiles 图片文件名，英文逗号分隔，如 "file1,file2,..."
      */
-    private void notifyImageService(String imageTitle) {
-        if (StringUtils.isEmpty(imageTitle)) {
+    private void notifyImageService(String imageFiles, String imageSavePath, String imageBaseUrl,
+        String imageReferer) {
+        if (StringUtils.isBlank(imageFiles)) {
             return;
         }
-        new Thread(() -> downloadImagesByTitle(imageTitle)).start();
+        new Thread(() -> downloadImagesByFiles(imageFiles, imageSavePath, imageBaseUrl,
+            imageReferer)).start();
     }
 
-    private void downloadImagesByTitle(String imageTitle) {
-        if (!imageTitle.startsWith("img:::")) {
+    private void downloadImagesByFiles(String imageFiles, String imageSavePath,
+        String imageBaseUrl, String imageReferer) {
+        if (StringUtils.isBlank(imageFiles)) {
             return;
         }
         try {
-            Files.createDirectories(IMAGE_HOST_DIR);
-            String[] fileNames = imageTitle.substring("img:::".length()).split(",");
+            Path imageHostDir = resolveImageHostDir(imageSavePath);
+            String resolvedBaseUrl = resolveImageBaseUrl(imageBaseUrl);
+            String resolvedReferer = resolveImageReferer(imageBaseUrl, imageReferer);
+            Files.createDirectories(imageHostDir);
+            String[] fileNames = imageFiles.split(",");
             for (String fileName : fileNames) {
                 String trimmedName = fileName.trim();
                 if (StringUtils.isEmpty(trimmedName)) {
@@ -324,18 +367,52 @@ public class RssSyncReconciler implements Reconciler<RssSyncReconciler.Request>,
 
                 String encodedFileName = URLEncoder.encode(trimmedName, StandardCharsets.UTF_8)
                     .replace("+", "%20");
-                String downloadUrl = IMAGE_CDN_BASE_URL + encodedFileName;
-                Path targetPath = IMAGE_HOST_DIR.resolve(trimmedName);
+                String downloadUrl = resolvedBaseUrl + encodedFileName;
+                Path targetPath = imageHostDir.resolve(trimmedName);
 
-                try (InputStream inputStream = new URL(downloadUrl).openStream()) {
+                HttpURLConnection connection = (HttpURLConnection) new URL(downloadUrl)
+                    .openConnection();
+                if (StringUtils.isNotBlank(resolvedReferer)) {
+                    connection.setRequestProperty("Referer", resolvedReferer);
+                }
+                try (InputStream inputStream = connection.getInputStream()) {
                     Files.copy(inputStream, targetPath, StandardCopyOption.REPLACE_EXISTING);
+                } finally {
+                    connection.disconnect();
                 }
             }
         } catch (Exception e) {
-            log.warn("Failed to download images for title: {}", imageTitle, e);
+            log.warn("Failed to download images for files: {}", imageFiles, e);
         }
     }
 
+    private Path resolveImageHostDir(String imageSavePath) {
+        if (StringUtils.isBlank(imageSavePath)) {
+            return Path.of(DEFAULT_IMAGE_HOST_PATH);
+        }
+        return Path.of(imageSavePath.trim());
+    }
+
+    private String resolveImageBaseUrl(String imageBaseUrl) {
+        String resolvedBaseUrl = StringUtils.isBlank(imageBaseUrl)
+            ? DEFAULT_IMAGE_BASE_URL
+            : imageBaseUrl.trim();
+        if (!resolvedBaseUrl.endsWith("/")) {
+            return resolvedBaseUrl + "/";
+        }
+        return resolvedBaseUrl;
+    }
+
+    private String resolveImageReferer(String imageBaseUrl, String imageReferer) {
+        if (StringUtils.isBlank(imageBaseUrl)) {
+            return DEFAULT_IMAGE_REFERER;
+        }
+        if (StringUtils.isBlank(imageReferer)) {
+            return null;
+        }
+        return imageReferer.trim();
+    }
+
     static int pageNullSafe(Integer page) {
         return ObjectUtils.defaultIfNull(page, 1);
     }
@@ -375,10 +452,16 @@ public class RssSyncReconciler implements Reconciler<RssSyncReconciler.Request>,
 
     @EventListener(RssFeedSyncEvent.class)
     public void onReplyEvent(RssFeedSyncEvent event) {
-        var request = new Request(event.getLink(), event.getSum(), event.getDisableSyncList());
+        var request = new Request(event.getLink(), event.getSum(), event.getDisableSyncList(),
+            event.isEnableWeiboImageRecordAndDownload(),
+            event.getImageSavePath(), event.getImageBaseUrl(), event.getImageReferer(),
+            event.getParallelCount());
         queue.addImmediately(request);
     }
 
-    public record Request(Link link, int sum, List<String> disableSyncList) {
+    public record Request(Link link, int sum, List<String> disableSyncList,
+                          boolean enableWeiboImageRecordAndDownload,
+                          String imageSavePath, String imageBaseUrl,
+                          String imageReferer, int parallelCount) {
     }
 }

src/main/java/la/moony/friends/service/impl/RssFeedSyncLogServiceImpl.java

@@ -74,14 +74,25 @@ public class RssFeedSyncLogServiceImpl implements RssFeedSyncLogService {
         CronFriendPost cron =  new CronFriendPost();
         CronFriendPost.CronSpec cronSpec = new CronFriendPost.CronSpec();
         cronSpec.setSuccessfulRetainLimit(5);
+        cronSpec.setParallelCount(1);
+        cronSpec.setEnableWeiboImageRecordAndDownload(false);
         cronSpec.setDisableSyncList(new ArrayList<>());
+        cronSpec.setImageSavePath("");
+        cronSpec.setImageBaseUrl("");
+        cronSpec.setImageReferer("");
         cron.setSpec(cronSpec);
         return client.fetch(CronFriendPost.class, "cron-friends-default")
             .defaultIfEmpty(cron)
             .flatMap(cronFriendPost -> {
                 var spec = cronFriendPost.getSpec();
                 int successfulRetainLimit = spec.getSuccessfulRetainLimit();
+                int parallelCount = spec.getParallelCount();
+                boolean enableWeiboImageRecordAndDownload =
+                    spec.isEnableWeiboImageRecordAndDownload();
                 List<String> disableSyncList = spec.getDisableSyncList();
+                String imageSavePath = spec.getImageSavePath();
+                String imageBaseUrl = spec.getImageBaseUrl();
+                String imageReferer = spec.getImageReferer();
                 int sum = successfulRetainLimit == 0 ? 5 : successfulRetainLimit;
                 var listOptions = new ListOptions();
                 FieldSelector fieldSelector = FieldSelector.of(isNull("metadata.deletionTimestamp"));
@@ -90,7 +101,11 @@ public class RssFeedSyncLogServiceImpl implements RssFeedSyncLogService {
                 }
                 listOptions.setFieldSelector(fieldSelector);
                 return client.listAll(Link.class, listOptions, Sort.by("metadata.creationTimestamp"))
-                    .doOnNext(link -> eventPublisher.publishEvent(new RssFeedSyncEvent(this, link,sum,disableSyncList)))
+                    .doOnNext(link -> eventPublisher.publishEvent(
+                        new RssFeedSyncEvent(this, link, sum, parallelCount,
+                            enableWeiboImageRecordAndDownload, disableSyncList, imageSavePath,
+                            imageBaseUrl, imageReferer)
+                    ))
                     .then();
             });
     }

ui/src/api/generated/models/cron-spec.ts

@@ -32,6 +32,36 @@ export interface CronSpec {
      * @memberof CronSpec
      */
     'disableSyncList'?: Array<string>;
+    /**
+     * 
+     * @type {number}
+     * @memberof CronSpec
+     */
+    'parallelCount'?: number;
+    /**
+     * 
+     * @type {boolean}
+     * @memberof CronSpec
+     */
+    'enableWeiboImageRecordAndDownload'?: boolean;
+    /**
+     * 
+     * @type {string}
+     * @memberof CronSpec
+     */
+    'imageSavePath'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof CronSpec
+     */
+    'imageBaseUrl'?: string;
+    /**
+     * 
+     * @type {string}
+     * @memberof CronSpec
+     */
+    'imageReferer'?: string;
     /**
      * 
      * @type {number}

ui/src/api/generated/models/friend-post-spec.ts

@@ -38,6 +38,12 @@ export interface FriendPostSpec {
      * @memberof FriendPostSpec
      */
     'description'?: string;
+    /**
+     * Comma-separated image file names.
+     * @type {string}
+     * @memberof FriendPostSpec
+     */
+    'imageFiles'?: string;
     /**
      * 
      * @type {string}

ui/src/views/CronTab.vue

@@ -18,7 +18,12 @@ const initialFormState: CronFriendPost = {
     cron: "@daily",
     timezone:"Asia/Shanghai",
     suspend: false,
+    enableWeiboImageRecordAndDownload: false,
+    parallelCount: 1,
     successfulRetainLimit: 0,
+    imageSavePath: "",
+    imageBaseUrl: "",
+    imageReferer: "",
     disableSyncList: undefined,
   },
   kind: "CronFriendPost",
@@ -130,6 +135,20 @@ const { mutate:save, isLoading:saveIsLoading } = useMutation({
                },
             ]"
           />
+          <FormKit
+            type="number"
+            name="parallelCount"
+            label="并行请求数量"
+            number="integer"
+            validation="required|number|min:1"
+            help="默认 1（串行），大于 1 为并行"
+          />
+          <FormKit
+            type="checkbox"
+            name="enableWeiboImageRecordAndDownload"
+            label="记录并下载微博图片"
+            help="默认关闭，开启后才会记录 imageFiles 字段并下载微博图片"
+          />
           <FormKit
             type="number"
             name="successfulRetainLimit"
@@ -138,6 +157,24 @@ const { mutate:save, isLoading:saveIsLoading } = useMutation({
             validation="required|number|min:0"
             help="设置之后会保留的数据条数，设置为 0 即为5条"
           />
+          <FormKit
+            type="text"
+            name="imageSavePath"
+            label="图片保存路径"
+            help="为空时使用默认路径 /root/.halo2/attachments/upload/image-host/"
+          />
+          <FormKit
+            type="text"
+            name="imageBaseUrl"
+            label="图片下载 Base URL"
+            help="为空时默认 https://wx3.sinaimg.cn/middle/"
+          />
+          <FormKit
+            type="text"
+            name="imageReferer"
+            label="图片下载 Referer"
+            help="当 Base URL 为空时默认 https://weibo.com/；当 Base URL 不为空时可留空"
+          />
           <LinkFormKit
             name="disableSyncList"
             label="禁止同步名单"