From 1893c73479c2ea011b4deb1788174061a2bc9562 Mon Sep 17 00:00:00 2001 From: Bane Stojkovic Date: Wed, 14 Oct 2026 16:52:08 +0200 Subject: [PATCH] cloud sync, add link to the owner page --- diylc/cloud-sync/README.md | 10 +++- diylc/cloud-sync/cloud-sync.example.json | 1 + .../plugins/cloud/sync/CloudSyncConfig.java | 10 +++- .../plugins/cloud/sync/CloudSyncJob.java | 58 +++++++++++++++++-- .../swing/plugins/cloud/sync/SyncState.java | 12 +++- .../plugins/cloud/sync/CloudSyncJobTests.java | 58 ++++++++++++++++++- 6 files changed, 132 insertions(+), 17 deletions(-) diff --git a/diylc/cloud-sync/README.md b/diylc/cloud-sync/README.md index a32bb7ea..7ed5db93 100644 --- a/diylc/cloud-sync/README.md +++ b/diylc/cloud-sync/README.md @@ -15,13 +15,17 @@ For each user: - `username`, `token`, `machineId`: the cloud API authenticates with the username (or email), not the numeric ID, plus a login token issued for that machine ID. - `userId`: names the user's folder under `storageDir`. +- `sourceUrl`: optional page cited as the source in project descriptions, instead of the link to + each file. - `category`, `keywords`: defaults for the user's pages. Category is the name shown in the upload dialog; the leading `- ` of subcategories is optional. - `pages`: page URLs. Use an object with `url`, `category` and `keywords` to override the defaults for one page. The project name comes from the file name, and the description from the file's own description -followed by a `Source:` link. +followed by a `Source:` link to the user's `sourceUrl`, or to the file when there is none. When +the source changes, the next run rewrites the `Source:` line of every project it cites, keeping the +rest of the description and the other project details as they are in the cloud. ## Running @@ -32,6 +36,6 @@ cloud-sync/cloud-sync.sh # sync for real ``` `storage//` mirrors the downloaded files with their thumbnails (`*.diy.png`), and keeps -`sync-state.json`, which maps each source URL to its cloud project ID and the hash that was last -uploaded. If the state is lost, the next run takes over the user's existing projects by name +`sync-state.json`, which maps each source URL to its cloud project ID, the hash that was last +uploaded and the source its description cites. If the state is lost, the next run takes over the user's existing projects by name instead of uploading duplicates. The exit status is non-zero if any page or file failed. diff --git a/diylc/cloud-sync/cloud-sync.example.json b/diylc/cloud-sync/cloud-sync.example.json index 77f466f9..94816ba7 100644 --- a/diylc/cloud-sync/cloud-sync.example.json +++ b/diylc/cloud-sync/cloud-sync.example.json @@ -8,6 +8,7 @@ "token": "login token issued to this username and machineId", "category": "Guitar Amplifiers", "keywords": "", + "sourceUrl": "https://example.com/", "pages": [ "https://example.com/layouts.htm", { diff --git a/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/CloudSyncConfig.java b/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/CloudSyncConfig.java index 895f83c4..f36ad4f4 100644 --- a/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/CloudSyncConfig.java +++ b/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/CloudSyncConfig.java @@ -23,6 +23,7 @@ package org.diylc.swing.plugins.cloud.sync; import java.io.File; import java.io.IOException; +import java.net.URI; import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; @@ -59,10 +60,15 @@ public record CloudSyncConfig(String serviceUrl, String storageDir, List pages) { + String category, String keywords, String sourceUrl, List pages) { + + public String sourceFor(URI file) { + return sourceUrl == null || sourceUrl.isBlank() ? file.toString() : sourceUrl; + } public String categoryFor(PageConfig page) { return page.category() == null ? category : page.category(); diff --git a/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/CloudSyncJob.java b/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/CloudSyncJob.java index b84d0e87..c5115ed7 100644 --- a/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/CloudSyncJob.java +++ b/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/CloudSyncJob.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.regex.Pattern; import javax.imageio.ImageIO; import org.apache.log4j.BasicConfigurator; @@ -65,7 +66,8 @@ import org.diylc.swing.plugins.cloud.sync.SyncState.SyncedFile; * Command-line job that mirrors .diy files published on users' own web pages into the DIYLC * cloud. Each new file is uploaded as a project and each changed file replaces the files of the * project it was uploaded as, through the same API calls the Upload action makes on the user's - * behalf. Meant to run headless from cron: + * behalf. The description of each project cites where the file came from, and is updated when the + * configured source changes. Meant to run headless from cron: * * 
  * java -cp diylc.jar org.diylc.swing.plugins.cloud.sync.CloudSyncJob cloud-sync.json [--dry-run]
@@ -88,6 +90,7 @@ public class CloudSyncJob {
   // column sizes of diylc_project on the server
   private static final int MAX_NAME_LENGTH = 50;
   private static final int MAX_DESCRIPTION_LENGTH = 1024;
+  private static final Pattern SOURCE_LINE = Pattern.compile("(?:\\A|\\s+)Source: \\S*\\s*\\z");
 
   private final CloudSyncConfig config;
   private final File storageDir;
@@ -223,8 +226,19 @@ public class CloudSyncJob {
     byte[] content = scraper.download(uri);
     String sha256 = sha256(content);
     SyncedFile synced = state.get(url);
+    String source = user.sourceFor(uri);
     if (synced != null && synced.sha256().equals(sha256)) {
-      return Outcome.UNCHANGED;
+      if (source.equals(synced.source())) {
+        return Outcome.UNCHANGED;
+      }
+      if (dryRun) {
+        LOG.info("Would cite " + source + " in project " + synced.projectId());
+      } else {
+        citeSource(user, synced.projectId(), source);
+        state.put(new SyncedFile(url, synced.projectId(), sha256, source,
+            Instant.now().toString()));
+      }
+      return Outcome.SOURCE_UPDATED;
     }
 
     File projectFile = localFile(userDir, uri);
@@ -247,7 +261,7 @@ public class CloudSyncJob {
 
     if (projectId == null) {
       checkResponse(service.uploadProject(user.username(), user.token(), user.machineId(), name,
-          resolveCategory(user.categoryFor(page)), description(projectDescription, uri),
+          resolveCategory(user.categoryFor(page)), description(projectDescription, source),
           diylcVersion, user.keywordsFor(page), thumbnailFile, projectFile, null));
       // the API does not return the ID of the new project, so the newest unclaimed one with the
       // same name is taken to be it
@@ -260,8 +274,12 @@ public class CloudSyncJob {
       checkResponse(service.replaceProjectFiles(user.username(), user.token(),
           user.machineId(), diylcVersion, thumbnailFile, projectFile, projectId));
       LOG.info("Updated project " + projectId + " from " + url);
+      // replacing the files leaves the description as it was, including the source it cites
+      if (synced == null || !source.equals(synced.source())) {
+        citeSource(user, projectId, source);
+      }
     }
-    state.put(new SyncedFile(url, projectId, sha256, Instant.now().toString()));
+    state.put(new SyncedFile(url, projectId, sha256, source, Instant.now().toString()));
     return outcome;
   }
 
@@ -284,6 +302,26 @@ public class CloudSyncJob {
     return presenter.getCurrentProject().getDescription();
   }
 
+  /**
+   * Points the source line of the project's description at the specified source, keeping the rest
+   * of the project details as they are in the cloud.
+   */
+  private void citeSource(UserConfig user, int projectId, String source) throws CloudException {
+    ProjectEntity project = getUploads(user).stream()
+        .filter(p -> p.getId() == projectId)
+        .findFirst()
+        .orElseThrow(() -> new CloudException(
+            "Project " + projectId + " is not among the user's projects"));
+    String description = withSource(project.getDescription(), source);
+    if (description.equals(project.getDescription())) {
+      return;
+    }
+    checkResponse(service.uploadProject(user.username(), user.token(), user.machineId(),
+        project.getName(), project.getCategoryForDisplay(), description, diylcVersion,
+        project.getKeywords(), null, null, projectId));
+    LOG.info("Cited " + source + " in project " + projectId);
+  }
+
   private List getUploads(UserConfig user) throws CloudException {
     if (uploads == null) {
       uploads = isBlank(user.username()) ? List.of() : fetchUploads(user);
@@ -350,7 +388,7 @@ public class CloudSyncJob {
     return name.length() > MAX_NAME_LENGTH ? name.substring(0, MAX_NAME_LENGTH).trim() : name;
   }
 
-  static String description(String projectDescription, URI source) {
+  static String description(String projectDescription, String source) {
     String sourceLine = "Source: " + source;
     if (isBlank(projectDescription)) {
       return sourceLine;
@@ -363,6 +401,14 @@ public class CloudSyncJob {
     return text + "\n\n" + sourceLine;
   }
 
+  // replaces the source line that description() ends with, or adds one when there is none, and
+  // keeps the rest of the description, which may have been edited in the cloud
+  static String withSource(String description, String source) {
+    String text =
+        description == null ? null : SOURCE_LINE.matcher(description).replaceFirst("");
+    return description(text, source);
+  }
+
   static File localFile(File userDir, URI uri) throws IOException {
     File root = userDir.getCanonicalFile();
     File file = new File(new File(root, uri.getHost()), uri.getPath()).getCanonicalFile();
@@ -381,7 +427,7 @@ public class CloudSyncJob {
   }
 
   private enum Outcome {
-    UNCHANGED, UPLOADED, UPDATED, FAILED
+    UNCHANGED, UPLOADED, UPDATED, SOURCE_UPDATED, FAILED
   }
 
   /**
diff --git a/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/SyncState.java b/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/SyncState.java
index 03499bf0..f3fef8a5 100644
--- a/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/SyncState.java
+++ b/diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/SyncState.java
@@ -34,8 +34,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 
 /**
- * Remembers, per source URL, which cloud project a file was uploaded as and the hash of the content
- * that was uploaded, so that later runs only upload what is new or changed.
+ * Remembers, per source URL, which cloud project a file was uploaded as, the hash of the content
+ * that was uploaded and the source its description cites, so that later runs only update what is
+ * new or changed.
  */
 public class SyncState {
 
@@ -78,7 +79,12 @@ public class SyncState {
         StandardCopyOption.ATOMIC_MOVE);
   }
 
-  public record SyncedFile(String url, int projectId, String sha256, String syncedOn) {}
+  /**
+   * @param source what the project description cites as the source, null when it is not known,
+   *        as for files synced before it was recorded
+   */
+  public record SyncedFile(String url, int projectId, String sha256, String source,
+      String syncedOn) {}
 
   private record Snapshot(List files) {}
 }
diff --git a/diylc/diylc-swing/src/test/java/org/diylc/swing/plugins/cloud/sync/CloudSyncJobTests.java b/diylc/diylc-swing/src/test/java/org/diylc/swing/plugins/cloud/sync/CloudSyncJobTests.java
index 0783c98d..23e8f632 100644
--- a/diylc/diylc-swing/src/test/java/org/diylc/swing/plugins/cloud/sync/CloudSyncJobTests.java
+++ b/diylc/diylc-swing/src/test/java/org/diylc/swing/plugins/cloud/sync/CloudSyncJobTests.java
@@ -76,6 +76,14 @@ public class CloudSyncJobTests {
     assertEquals(1, cloud.projects.size());
   }
 
+  @Test
+  public void citesConfiguredSourcePageInsteadOfFile() throws Exception {
+    FakeCloud cloud = new FakeCloud();
+
+    assertEquals(0, newJob(cloud, new FakeScraper(emptyProject()), "https://example.com/").run());
+    assertEquals("Source: https://example.com/", cloud.lastDescription);
+  }
+
   @Test
   public void takesOverProjectAlreadyUploadedUnderSameName() throws Exception {
     FakeCloud cloud = new FakeCloud();
@@ -83,7 +91,26 @@ public class CloudSyncJobTests {
     cloud.add("Something Else");
 
     assertEquals(0, newJob(cloud, new FakeScraper(emptyProject())).run());
-    assertEquals(List.of("search", "replaceProjectFiles 1"), cloud.writesAndLookups);
+    assertEquals(List.of("search", "replaceProjectFiles 1", "updateProjectDetails 1"),
+        cloud.writesAndLookups);
+    assertEquals("Source: " + FILE, cloud.lastDescription);
+  }
+
+  @Test
+  public void changedSourceIsCitedInExistingProjects() throws Exception {
+    FakeCloud cloud = new FakeCloud();
+    FakeScraper scraper = new FakeScraper(emptyProject());
+    assertEquals(0, newJob(cloud, scraper).run());
+    cloud.projects.get(0).setDescription("Notes\n\nSource: " + FILE);
+
+    cloud.writesAndLookups.clear();
+    assertEquals(0, newJob(cloud, scraper, "https://example.com/").run());
+    assertEquals(List.of("search", "updateProjectDetails 1"), cloud.writesAndLookups);
+    assertEquals("Notes\n\nSource: https://example.com/", cloud.lastDescription);
+
+    cloud.writesAndLookups.clear();
+    assertEquals(0, newJob(cloud, scraper, "https://example.com/").run());
+    assertEquals(List.of(), cloud.writesAndLookups);
   }
 
   @Test
@@ -117,13 +144,25 @@ public class CloudSyncJobTests {
 
   @Test
   public void descriptionAlwaysCitesSource() {
-    URI source = URI.create("https://example.com/a.diy");
+    String source = "https://example.com/a.diy";
     assertEquals("Source: https://example.com/a.diy", CloudSyncJob.description(null, source));
     String description = CloudSyncJob.description("x".repeat(2000), source);
     assertEquals(1024, description.length());
     assertTrue(description.endsWith("\n\nSource: https://example.com/a.diy"));
   }
 
+  @Test
+  public void withSourceReplacesOnlyTheSourceLine() {
+    String source = "https://example.com/";
+    assertEquals("Notes\n\nSource: https://example.com/",
+        CloudSyncJob.withSource("Notes\r\n\r\nSource: https://example.com/a.diy\r\n", source));
+    assertEquals("Source: https://example.com/",
+        CloudSyncJob.withSource("Source: https://example.com/a.diy", source));
+    assertEquals("Notes\n\nSource: https://example.com/",
+        CloudSyncJob.withSource("Notes", source));
+    assertEquals("Source: https://example.com/", CloudSyncJob.withSource(null, source));
+  }
+
   @Test(expected = IOException.class)
   public void refusesToStoreOutsideUserDirectory() throws IOException {
     CloudSyncJob.localFile(new File("target/sync-test"),
@@ -144,8 +183,13 @@ public class CloudSyncJobTests {
   }
 
   private CloudSyncJob newJob(FakeCloud cloud, FakeScraper scraper) throws IOException {
+    return newJob(cloud, scraper, null);
+  }
+
+  private CloudSyncJob newJob(FakeCloud cloud, FakeScraper scraper, String sourceUrl)
+      throws IOException {
     UserConfig user = new UserConfig(7, "someone", "cron-job", "token", "Guitar Amplifiers", null,
-        List.of(PageConfig.of(PAGE.toString())));
+        sourceUrl, List.of(PageConfig.of(PAGE.toString())));
     CloudSyncConfig config = new CloudSyncConfig(null, null, List.of(user));
     return new CloudSyncJob(config, new File(folder.getRoot(), "storage"), false, cloud.api(),
         scraper);
@@ -188,6 +232,7 @@ public class CloudSyncJobTests {
       ProjectEntity project = new ProjectEntity();
       project.setId(projects.size() + 1);
       project.setName(name);
+      project.setCategoryForDisplay("- Guitar Amplifiers");
       projects.add(project);
     }
 
@@ -203,7 +248,14 @@ public class CloudSyncJobTests {
               case "uploadProject":
                 assertEquals("- Guitar Amplifiers", args[4]);
                 lastDescription = (String) args[5];
+                // a project ID means only the details of that project are updated
+                if (args[10] != null) {
+                  projects.get((Integer) args[10] - 1).setDescription(lastDescription);
+                  writesAndLookups.add("updateProjectDetails " + args[10]);
+                  return "Success";
+                }
                 add((String) args[3]);
+                projects.get(projects.size() - 1).setDescription(lastDescription);
                 writesAndLookups.add("uploadProject " + args[3]);
                 return "Success";
               case "replaceProjectFiles":