cloud sync, add link to the owner page

diylc/cloud-sync/README.md

@@ -15,13 +15,17 @@ For each user:
 - `username`, `token`, `machineId`: the cloud API authenticates with the username (or email), not
   the numeric ID, plus a login token issued for that machine ID.
 - `userId`: names the user's folder under `storageDir`.
+- `sourceUrl`: optional page cited as the source in project descriptions, instead of the link to
+  each file.
 - `category`, `keywords`: defaults for the user's pages. Category is the name shown in the upload
   dialog; the leading `- ` of subcategories is optional.
 - `pages`: page URLs. Use an object with `url`, `category` and `keywords` to override the defaults
   for one page.
 
 The project name comes from the file name, and the description from the file's own description
-followed by a `Source:` link.
+followed by a `Source:` link to the user's `sourceUrl`, or to the file when there is none. When
+the source changes, the next run rewrites the `Source:` line of every project it cites, keeping the
+rest of the description and the other project details as they are in the cloud.
 
 ## Running
 
@@ -32,6 +36,6 @@ cloud-sync/cloud-sync.sh                          # sync for real
 ```
 
 `storage/<userId>/` mirrors the downloaded files with their thumbnails (`*.diy.png`), and keeps
-`sync-state.json`, which maps each source URL to its cloud project ID and the hash that was last
-uploaded. If the state is lost, the next run takes over the user's existing projects by name
+`sync-state.json`, which maps each source URL to its cloud project ID, the hash that was last
+uploaded and the source its description cites. If the state is lost, the next run takes over the user's existing projects by name
 instead of uploading duplicates. The exit status is non-zero if any page or file failed.

diylc/cloud-sync/cloud-sync.example.json

@@ -8,6 +8,7 @@
       "token": "login token issued to this username and machineId",
       "category": "Guitar Amplifiers",
       "keywords": "",
+      "sourceUrl": "https://example.com/",
       "pages": [
         "https://example.com/layouts.htm",
         {

diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/CloudSyncConfig.java

@@ -23,6 +23,7 @@ package org.diylc.swing.plugins.cloud.sync;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URI;
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -59,10 +60,15 @@ public record CloudSyncConfig(String serviceUrl, String storageDir, List<UserCon
 
   /**
    * The cloud API authenticates by username (or email), token and machine ID; {@code userId} only
-   * names the user's storage folder.
+   * names the user's storage folder. {@code sourceUrl}, when set, is the page cited as the source
+   * in project descriptions instead of the link to the file itself.
    */
   public record UserConfig(int userId, String username, String machineId, String token,
-      String category, String keywords, List<PageConfig> pages) {
+      String category, String keywords, String sourceUrl, List<PageConfig> pages) {
+
+    public String sourceFor(URI file) {
+      return sourceUrl == null || sourceUrl.isBlank() ? file.toString() : sourceUrl;
+    }
 
     public String categoryFor(PageConfig page) {
       return page.category() == null ? category : page.category();

diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/CloudSyncJob.java

@@ -38,6 +38,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.regex.Pattern;
 import javax.imageio.ImageIO;
 
 import org.apache.log4j.BasicConfigurator;
@@ -65,7 +66,8 @@ import org.diylc.swing.plugins.cloud.sync.SyncState.SyncedFile;
  * Command-line job that mirrors .diy files published on users' own web pages into the DIYLC
  * cloud. Each new file is uploaded as a project and each changed file replaces the files of the
  * project it was uploaded as, through the same API calls the Upload action makes on the user's
- * behalf. Meant to run headless from cron:
+ * behalf. The description of each project cites where the file came from, and is updated when the
+ * configured source changes. Meant to run headless from cron:
  *
  * <pre>
  * java -cp diylc.jar org.diylc.swing.plugins.cloud.sync.CloudSyncJob cloud-sync.json [--dry-run]
@@ -88,6 +90,7 @@ public class CloudSyncJob {
   // column sizes of diylc_project on the server
   private static final int MAX_NAME_LENGTH = 50;
   private static final int MAX_DESCRIPTION_LENGTH = 1024;
+  private static final Pattern SOURCE_LINE = Pattern.compile("(?:\\A|\\s+)Source: \\S*\\s*\\z");
 
   private final CloudSyncConfig config;
   private final File storageDir;
@@ -223,8 +226,19 @@ public class CloudSyncJob {
     byte[] content = scraper.download(uri);
     String sha256 = sha256(content);
     SyncedFile synced = state.get(url);
+    String source = user.sourceFor(uri);
     if (synced != null && synced.sha256().equals(sha256)) {
-      return Outcome.UNCHANGED;
+      if (source.equals(synced.source())) {
+        return Outcome.UNCHANGED;
+      }
+      if (dryRun) {
+        LOG.info("Would cite " + source + " in project " + synced.projectId());
+      } else {
+        citeSource(user, synced.projectId(), source);
+        state.put(new SyncedFile(url, synced.projectId(), sha256, source,
+            Instant.now().toString()));
+      }
+      return Outcome.SOURCE_UPDATED;
     }
 
     File projectFile = localFile(userDir, uri);
@@ -247,7 +261,7 @@ public class CloudSyncJob {
 
     if (projectId == null) {
       checkResponse(service.uploadProject(user.username(), user.token(), user.machineId(), name,
-          resolveCategory(user.categoryFor(page)), description(projectDescription, uri),
+          resolveCategory(user.categoryFor(page)), description(projectDescription, source),
           diylcVersion, user.keywordsFor(page), thumbnailFile, projectFile, null));
       // the API does not return the ID of the new project, so the newest unclaimed one with the
       // same name is taken to be it
@@ -260,8 +274,12 @@ public class CloudSyncJob {
       checkResponse(service.replaceProjectFiles(user.username(), user.token(),
           user.machineId(), diylcVersion, thumbnailFile, projectFile, projectId));
       LOG.info("Updated project " + projectId + " from " + url);
+      // replacing the files leaves the description as it was, including the source it cites
+      if (synced == null || !source.equals(synced.source())) {
+        citeSource(user, projectId, source);
+      }
     }
-    state.put(new SyncedFile(url, projectId, sha256, Instant.now().toString()));
+    state.put(new SyncedFile(url, projectId, sha256, source, Instant.now().toString()));
     return outcome;
   }
 
@@ -284,6 +302,26 @@ public class CloudSyncJob {
     return presenter.getCurrentProject().getDescription();
   }
 
+  /**
+   * Points the source line of the project's description at the specified source, keeping the rest
+   * of the project details as they are in the cloud.
+   */
+  private void citeSource(UserConfig user, int projectId, String source) throws CloudException {
+    ProjectEntity project = getUploads(user).stream()
+        .filter(p -> p.getId() == projectId)
+        .findFirst()
+        .orElseThrow(() -> new CloudException(
+            "Project " + projectId + " is not among the user's projects"));
+    String description = withSource(project.getDescription(), source);
+    if (description.equals(project.getDescription())) {
+      return;
+    }
+    checkResponse(service.uploadProject(user.username(), user.token(), user.machineId(),
+        project.getName(), project.getCategoryForDisplay(), description, diylcVersion,
+        project.getKeywords(), null, null, projectId));
+    LOG.info("Cited " + source + " in project " + projectId);
+  }
+
   private List<ProjectEntity> getUploads(UserConfig user) throws CloudException {
     if (uploads == null) {
       uploads = isBlank(user.username()) ? List.of() : fetchUploads(user);
@@ -350,7 +388,7 @@ public class CloudSyncJob {
     return name.length() > MAX_NAME_LENGTH ? name.substring(0, MAX_NAME_LENGTH).trim() : name;
   }
 
-  static String description(String projectDescription, URI source) {
+  static String description(String projectDescription, String source) {
     String sourceLine = "Source: " + source;
     if (isBlank(projectDescription)) {
       return sourceLine;
@@ -363,6 +401,14 @@ public class CloudSyncJob {
     return text + "\n\n" + sourceLine;
   }
 
+  // replaces the source line that description() ends with, or adds one when there is none, and
+  // keeps the rest of the description, which may have been edited in the cloud
+  static String withSource(String description, String source) {
+    String text =
+        description == null ? null : SOURCE_LINE.matcher(description).replaceFirst("");
+    return description(text, source);
+  }
+
   static File localFile(File userDir, URI uri) throws IOException {
     File root = userDir.getCanonicalFile();
     File file = new File(new File(root, uri.getHost()), uri.getPath()).getCanonicalFile();
@@ -381,7 +427,7 @@ public class CloudSyncJob {
   }
 
   private enum Outcome {
-    UNCHANGED, UPLOADED, UPDATED, FAILED
+    UNCHANGED, UPLOADED, UPDATED, SOURCE_UPDATED, FAILED
   }
 
   /**

diylc/diylc-swing/src/main/java/org/diylc/swing/plugins/cloud/sync/SyncState.java

@@ -34,8 +34,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 
 /**
- * Remembers, per source URL, which cloud project a file was uploaded as and the hash of the content
- * that was uploaded, so that later runs only upload what is new or changed.
+ * Remembers, per source URL, which cloud project a file was uploaded as, the hash of the content
+ * that was uploaded and the source its description cites, so that later runs only update what is
+ * new or changed.
  */
 public class SyncState {
 
@@ -78,7 +79,12 @@ public class SyncState {
         StandardCopyOption.ATOMIC_MOVE);
   }
 
-  public record SyncedFile(String url, int projectId, String sha256, String syncedOn) {}
+  /**
+   * @param source what the project description cites as the source, null when it is not known,
+   *        as for files synced before it was recorded
+   */
+  public record SyncedFile(String url, int projectId, String sha256, String source,
+      String syncedOn) {}
 
   private record Snapshot(List<SyncedFile> files) {}
 }

diylc/diylc-swing/src/test/java/org/diylc/swing/plugins/cloud/sync/CloudSyncJobTests.java

@@ -76,6 +76,14 @@ public class CloudSyncJobTests {
     assertEquals(1, cloud.projects.size());
   }
 
+  @Test
+  public void citesConfiguredSourcePageInsteadOfFile() throws Exception {
+    FakeCloud cloud = new FakeCloud();
+
+    assertEquals(0, newJob(cloud, new FakeScraper(emptyProject()), "https://example.com/").run());
+    assertEquals("Source: https://example.com/", cloud.lastDescription);
+  }
+
   @Test
   public void takesOverProjectAlreadyUploadedUnderSameName() throws Exception {
     FakeCloud cloud = new FakeCloud();
@@ -83,7 +91,26 @@ public class CloudSyncJobTests {
     cloud.add("Something Else");
 
     assertEquals(0, newJob(cloud, new FakeScraper(emptyProject())).run());
-    assertEquals(List.of("search", "replaceProjectFiles 1"), cloud.writesAndLookups);
+    assertEquals(List.of("search", "replaceProjectFiles 1", "updateProjectDetails 1"),
+        cloud.writesAndLookups);
+    assertEquals("Source: " + FILE, cloud.lastDescription);
+  }
+
+  @Test
+  public void changedSourceIsCitedInExistingProjects() throws Exception {
+    FakeCloud cloud = new FakeCloud();
+    FakeScraper scraper = new FakeScraper(emptyProject());
+    assertEquals(0, newJob(cloud, scraper).run());
+    cloud.projects.get(0).setDescription("Notes\n\nSource: " + FILE);
+
+    cloud.writesAndLookups.clear();
+    assertEquals(0, newJob(cloud, scraper, "https://example.com/").run());
+    assertEquals(List.of("search", "updateProjectDetails 1"), cloud.writesAndLookups);
+    assertEquals("Notes\n\nSource: https://example.com/", cloud.lastDescription);
+
+    cloud.writesAndLookups.clear();
+    assertEquals(0, newJob(cloud, scraper, "https://example.com/").run());
+    assertEquals(List.of(), cloud.writesAndLookups);
   }
 
   @Test
@@ -117,13 +144,25 @@ public class CloudSyncJobTests {
 
   @Test
   public void descriptionAlwaysCitesSource() {
-    URI source = URI.create("https://example.com/a.diy");
+    String source = "https://example.com/a.diy";
     assertEquals("Source: https://example.com/a.diy", CloudSyncJob.description(null, source));
     String description = CloudSyncJob.description("x".repeat(2000), source);
     assertEquals(1024, description.length());
     assertTrue(description.endsWith("\n\nSource: https://example.com/a.diy"));
   }
 
+  @Test
+  public void withSourceReplacesOnlyTheSourceLine() {
+    String source = "https://example.com/";
+    assertEquals("Notes\n\nSource: https://example.com/",
+        CloudSyncJob.withSource("Notes\r\n\r\nSource: https://example.com/a.diy\r\n", source));
+    assertEquals("Source: https://example.com/",
+        CloudSyncJob.withSource("Source: https://example.com/a.diy", source));
+    assertEquals("Notes\n\nSource: https://example.com/",
+        CloudSyncJob.withSource("Notes", source));
+    assertEquals("Source: https://example.com/", CloudSyncJob.withSource(null, source));
+  }
+
   @Test(expected = IOException.class)
   public void refusesToStoreOutsideUserDirectory() throws IOException {
     CloudSyncJob.localFile(new File("target/sync-test"),
@@ -144,8 +183,13 @@ public class CloudSyncJobTests {
   }
 
   private CloudSyncJob newJob(FakeCloud cloud, FakeScraper scraper) throws IOException {
+    return newJob(cloud, scraper, null);
+  }
+
+  private CloudSyncJob newJob(FakeCloud cloud, FakeScraper scraper, String sourceUrl)
+      throws IOException {
     UserConfig user = new UserConfig(7, "someone", "cron-job", "token", "Guitar Amplifiers", null,
-        List.of(PageConfig.of(PAGE.toString())));
+        sourceUrl, List.of(PageConfig.of(PAGE.toString())));
     CloudSyncConfig config = new CloudSyncConfig(null, null, List.of(user));
     return new CloudSyncJob(config, new File(folder.getRoot(), "storage"), false, cloud.api(),
         scraper);
@@ -188,6 +232,7 @@ public class CloudSyncJobTests {
       ProjectEntity project = new ProjectEntity();
       project.setId(projects.size() + 1);
       project.setName(name);
+      project.setCategoryForDisplay("- Guitar Amplifiers");
       projects.add(project);
     }
 
@@ -203,7 +248,14 @@ public class CloudSyncJobTests {
               case "uploadProject":
                 assertEquals("- Guitar Amplifiers", args[4]);
                 lastDescription = (String) args[5];
+                // a project ID means only the details of that project are updated
+                if (args[10] != null) {
+                  projects.get((Integer) args[10] - 1).setDescription(lastDescription);
+                  writesAndLookups.add("updateProjectDetails " + args[10]);
+                  return "Success";
+                }
                 add((String) args[3]);
+                projects.get(projects.size() - 1).setDescription(lastDescription);
                 writesAndLookups.add("uploadProject " + args[3]);
                 return "Success";
               case "replaceProjectFiles":