Cloud project scraper added

2026-11-13 02:32:09 +10:00 · 2026-10-13 16:04:51 +02:00
parent 84726e6d16
commit 1ec8805c91
10 changed files with 973 additions and 0 deletions
@@ -0,0 +1,91 @@
+/*
+
+    DIY Layout Creator (DIYLC).
+    Copyright (c) 2009-2025 held jointly by the individual authors.
+
+    This file is part of DIYLC.
+
+    DIYLC is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    DIYLC is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with DIYLC.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+package org.diylc.swing.plugins.cloud.sync;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+/**
+ * Configuration of {@link CloudSyncJob}. It carries login tokens, so the file it is read from is
+ * kept out of version control.
+ *
+ * @param serviceUrl optional override of the cloud API base URL
+ * @param storageDir where downloaded files, thumbnails and sync state are kept, resolved against
+ *        the directory of the configuration file when relative
+ * @param users accounts whose pages are synced
+ */
+public record CloudSyncConfig(String serviceUrl, String storageDir, List<UserConfig> users) {
+
+  public static final String DEFAULT_SERVICE_URL = "http://www.diy-fever.com/diylc/api/v1";
+  public static final String DEFAULT_STORAGE_DIR = "storage";
+
+  public static CloudSyncConfig load(File file) throws IOException {
+    return new ObjectMapper().readValue(file, CloudSyncConfig.class);
+  }
+
+  public String serviceUrlOrDefault() {
+    return serviceUrl == null || serviceUrl.isBlank() ? DEFAULT_SERVICE_URL : serviceUrl;
+  }
+
+  public File resolveStorageDir(File configFile) {
+    File dir =
+        new File(storageDir == null || storageDir.isBlank() ? DEFAULT_STORAGE_DIR : storageDir);
+    return dir.isAbsolute() ? dir
+        : new File(configFile.getAbsoluteFile().getParentFile(), dir.getPath());
+  }
+
+  /**
+   * The cloud API authenticates by username (or email), token and machine ID; {@code userId} only
+   * names the user's storage folder.
+   */
+  public record UserConfig(int userId, String username, String machineId, String token,
+      String category, String keywords, List<PageConfig> pages) {
+
+    public String categoryFor(PageConfig page) {
+      return page.category() == null ? category : page.category();
+    }
+
+    public String keywordsFor(PageConfig page) {
+      String result = page.keywords() == null ? keywords : page.keywords();
+      return result == null ? "" : result;
+    }
+  }
+
+  /**
+   * A page to scrape for links to .diy files. A bare URL string in the JSON is accepted as a page
+   * that inherits category and keywords from its user.
+   */
+  public record PageConfig(String url, String category, String keywords) {
+
+    @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
+    public PageConfig {}
+
+    @JsonCreator(mode = JsonCreator.Mode.DELEGATING)
+    public static PageConfig of(String url) {
+      return new PageConfig(url, null, null);
+    }
+  }
+}
@@ -0,0 +1,408 @@
+/*
+
+    DIY Layout Creator (DIYLC).
+    Copyright (c) 2009-2025 held jointly by the individual authors.
+
+    This file is part of DIYLC.
+
+    DIYLC is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    DIYLC is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with DIYLC.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+package org.diylc.swing.plugins.cloud.sync;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.HexFormat;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import javax.imageio.ImageIO;
+
+import org.apache.log4j.BasicConfigurator;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+import com.diyfever.httpproxy.PhpFlatProxy;
+import com.diyfever.httpproxy.ProxyFactory;
+
+import org.diylc.appframework.miscutils.ConfigurationManager;
+import org.diylc.appframework.miscutils.InMemoryConfigurationManager;
+import org.diylc.common.DummyView;
+import org.diylc.core.IView;
+import org.diylc.plugins.cloud.model.IServiceAPI;
+import org.diylc.plugins.cloud.model.ProjectEntity;
+import org.diylc.plugins.cloud.service.CloudException;
+import org.diylc.presenter.Presenter;
+import org.diylc.swing.plugins.cloud.ThumbnailGenerator;
+import org.diylc.swing.plugins.cloud.sync.CloudSyncConfig.PageConfig;
+import org.diylc.swing.plugins.cloud.sync.CloudSyncConfig.UserConfig;
+import org.diylc.swing.plugins.cloud.sync.SyncState.SyncedFile;
+
+/**
+ * Command-line job that mirrors .diy files published on users' own web pages into the DIYLC
+ * cloud. Each new file is uploaded as a project and each changed file replaces the files of the
+ * project it was uploaded as, through the same API calls the Upload action makes on the user's
+ * behalf. Meant to run headless from cron:
+ *
+ * <pre>
+ * java -cp diylc.jar org.diylc.swing.plugins.cloud.sync.CloudSyncJob cloud-sync.json [--dry-run]
+ * </pre>
+ *
+ * With {@code --dry-run} files are downloaded and thumbnails rendered into local storage, but
+ * nothing is uploaded and the sync state is left untouched.
+ *
+ * @author Branislav Stojkovic
+ */
+public class CloudSyncJob {
+
+  private static final Logger LOG = Logger.getLogger(CloudSyncJob.class);
+
+  private static final String DRY_RUN = "--dry-run";
+  private static final String SUCCESS = "Success";
+  private static final String STATE_FILE = "sync-state.json";
+  private static final String CONFIGURATION_NAME = "diylc-cloud-sync";
+  private static final String SORT_BY_AGE = "Age";
+  // column sizes of diylc_project on the server
+  private static final int MAX_NAME_LENGTH = 50;
+  private static final int MAX_DESCRIPTION_LENGTH = 1024;
+
+  private final CloudSyncConfig config;
+  private final File storageDir;
+  private final boolean dryRun;
+  private final IServiceAPI service;
+  private final PageScraper scraper;
+  private final RecordingView view = new RecordingView();
+  private final Presenter presenter;
+  private final ThumbnailGenerator thumbnailGenerator;
+  private final String diylcVersion;
+  private List<String> categories;
+  // projects the current user already has in the cloud, fetched only once a file is found that
+  // the sync state does not know
+  private List<ProjectEntity> uploads;
+
+  public CloudSyncJob(CloudSyncConfig config, File storageDir, boolean dryRun) {
+    this(config, storageDir, dryRun, new ProxyFactory(new PhpFlatProxy())
+        .createProxy(IServiceAPI.class, config.serviceUrlOrDefault()), new PageScraper());
+  }
+
+  CloudSyncJob(CloudSyncConfig config, File storageDir, boolean dryRun, IServiceAPI service,
+      PageScraper scraper) {
+    this.config = config;
+    this.storageDir = storageDir;
+    this.dryRun = dryRun;
+    this.service = service;
+    this.scraper = scraper;
+    this.presenter = new Presenter(view, InMemoryConfigurationManager.getInstance());
+    this.thumbnailGenerator = new ThumbnailGenerator(presenter);
+    this.diylcVersion = presenter.getCurrentVersionNumber().toString();
+  }
+
+  public static void main(String[] args) {
+    System.setProperty("java.awt.headless", "true");
+    // replaces the application's log4j.properties, which log4j picks up from the classpath
+    LogManager.resetConfiguration();
+    BasicConfigurator.configure();
+    Logger.getRootLogger().setLevel(Level.WARN);
+    LOG.setLevel(Level.INFO);
+
+    List<String> arguments = new ArrayList<>(Arrays.asList(args));
+    boolean dryRun = arguments.remove(DRY_RUN);
+    if (arguments.size() != 1) {
+      System.err.println("Usage: CloudSyncJob <config.json> [" + DRY_RUN + "]");
+      System.exit(2);
+    }
+
+    // components read drawing options from the global configuration, so it needs initializing;
+    // a name of its own keeps the job on default settings and away from the user's config
+    ConfigurationManager.getInstance().initialize(CONFIGURATION_NAME);
+
+    int failures;
+    try {
+      File configFile = new File(arguments.get(0));
+      CloudSyncConfig config = CloudSyncConfig.load(configFile);
+      failures = new CloudSyncJob(config, config.resolveStorageDir(configFile), dryRun).run();
+    } catch (Exception e) {
+      LOG.error("Cloud sync failed", e);
+      failures = 1;
+    }
+    System.exit(failures == 0 ? 0 : 1);
+  }
+
+  /**
+   * @return the number of pages and files that could not be synced
+   */
+  public int run() throws CloudException {
+    categories = service.getCategories();
+    if (categories == null || categories.isEmpty()) {
+      throw new CloudException("Could not fetch categories from the server.");
+    }
+    int failures = 0;
+    for (UserConfig user : config.users()) {
+      failures += syncUser(user);
+    }
+    return failures;
+  }
+
+  private int syncUser(UserConfig user) {
+    File userDir = new File(storageDir, Integer.toString(user.userId()));
+    Map<Outcome, Integer> outcomes = new EnumMap<>(Outcome.class);
+    try {
+      validate(user);
+      SyncState state = SyncState.load(new File(userDir, STATE_FILE));
+      uploads = null;
+      Set<URI> seen = new HashSet<>();
+      for (PageConfig page : user.pages()) {
+        List<URI> links;
+        try {
+          links = scraper.findDiyLinks(URI.create(page.url()));
+        } catch (Exception e) {
+          LOG.error("Could not scrape " + page.url(), e);
+          outcomes.merge(Outcome.FAILED, 1, Integer::sum);
+          continue;
+        }
+        for (URI link : links) {
+          if (seen.add(link)) {
+            Outcome outcome;
+            try {
+              outcome = syncFile(user, page, link, userDir, state);
+            } catch (Exception e) {
+              LOG.error("Could not sync " + link, e);
+              outcome = Outcome.FAILED;
+            }
+            outcomes.merge(outcome, 1, Integer::sum);
+          }
+        }
+      }
+    } catch (Exception e) {
+      LOG.error("Could not sync user " + user.userId(), e);
+      outcomes.merge(Outcome.FAILED, 1, Integer::sum);
+    }
+    LOG.info("User " + user.userId() + (dryRun ? " (dry run): " : ": ") + outcomes);
+    return outcomes.getOrDefault(Outcome.FAILED, 0);
+  }
+
+  private void validate(UserConfig user) throws CloudException {
+    if (!dryRun && (isBlank(user.username()) || isBlank(user.token())
+        || isBlank(user.machineId()))) {
+      throw new CloudException("username, token and machineId are required");
+    }
+    if (user.pages() == null || user.pages().isEmpty()) {
+      throw new CloudException("No pages configured");
+    }
+    for (PageConfig page : user.pages()) {
+      resolveCategory(user.categoryFor(page));
+    }
+  }
+
+  private Outcome syncFile(UserConfig user, PageConfig page, URI uri, File userDir,
+      SyncState state) throws Exception {
+    String url = uri.toString();
+    byte[] content = scraper.download(uri);
+    String sha256 = sha256(content);
+    SyncedFile synced = state.get(url);
+    if (synced != null && synced.sha256().equals(sha256)) {
+      return Outcome.UNCHANGED;
+    }
+
+    File projectFile = localFile(userDir, uri);
+    projectFile.getParentFile().mkdirs();
+    Files.write(projectFile.toPath(), content);
+    File thumbnailFile = new File(projectFile.getPath() + ".png");
+    String projectDescription = renderThumbnail(projectFile, thumbnailFile);
+    String name = projectName(uri);
+
+    // a project already in the cloud under the same name, uploaded by hand or by a run whose
+    // state was lost, is taken over rather than duplicated
+    Integer projectId = synced != null ? Integer.valueOf(synced.projectId())
+        : findUnclaimed(getUploads(user), name, state, Comparator.naturalOrder()).orElse(null);
+    Outcome outcome = projectId == null ? Outcome.UPLOADED : Outcome.UPDATED;
+    if (dryRun) {
+      LOG.info((projectId == null ? "Would upload " : "Would update project " + projectId
+          + " from ") + url);
+      return outcome;
+    }
+
+    if (projectId == null) {
+      checkResponse(service.uploadProject(user.username(), user.token(), user.machineId(), name,
+          resolveCategory(user.categoryFor(page)), description(projectDescription, uri),
+          diylcVersion, user.keywordsFor(page), thumbnailFile, projectFile, null));
+      // the API does not return the ID of the new project, so the newest unclaimed one with the
+      // same name is taken to be it
+      uploads = fetchUploads(user);
+      projectId = findUnclaimed(uploads, name, state, Comparator.reverseOrder())
+          .orElseThrow(() -> new CloudException(
+              "Uploaded " + url + " but could not find it among the user's projects"));
+      LOG.info("Uploaded " + url + " as project " + projectId);
+    } else {
+      checkResponse(service.replaceProjectFiles(user.username(), user.token(),
+          user.machineId(), diylcVersion, thumbnailFile, projectFile, projectId));
+      LOG.info("Updated project " + projectId + " from " + url);
+    }
+    state.put(new SyncedFile(url, projectId, sha256, Instant.now().toString()));
+    return outcome;
+  }
+
+  /**
+   * Loads the project into the off-screen presenter and writes its thumbnail.
+   *
+   * @return the project description stored in the file
+   */
+  private String renderThumbnail(File projectFile, File thumbnailFile)
+      throws IOException, CloudException {
+    view.takeError();
+    presenter.loadProjectFromFile(projectFile.getAbsolutePath());
+    String error = view.takeError();
+    if (error != null) {
+      throw new CloudException(error);
+    }
+    if (!ImageIO.write(thumbnailGenerator.getThumbnail(), "png", thumbnailFile)) {
+      throw new IOException("Could not write " + thumbnailFile);
+    }
+    return presenter.getCurrentProject().getDescription();
+  }
+
+  private List<ProjectEntity> getUploads(UserConfig user) throws CloudException {
+    if (uploads == null) {
+      uploads = isBlank(user.username()) ? List.of() : fetchUploads(user);
+    }
+    return uploads;
+  }
+
+  private List<ProjectEntity> fetchUploads(UserConfig user) throws CloudException {
+    Object res =
+        service.search("", "", 1, Integer.MAX_VALUE, SORT_BY_AGE, user.username(), null);
+    if (res instanceof List<?> list) {
+      return list.stream().map(ProjectEntity.class::cast).toList();
+    }
+    // a single result is not wrapped in a list, and no results come back as an empty string
+    if (res instanceof ProjectEntity project) {
+      return List.of(project);
+    }
+    if (res == null || res instanceof String s && s.isBlank()) {
+      return List.of();
+    }
+    throw new CloudException("Unexpected response while fetching user projects: " + res);
+  }
+
+  private static Optional<Integer> findUnclaimed(List<ProjectEntity> uploads, String name,
+      SyncState state, Comparator<Integer> preference) {
+    return uploads.stream()
+        .filter(p -> name.equals(p.getName()) && !state.isClaimed(p.getId()))
+        .map(ProjectEntity::getId)
+        .min(preference);
+  }
+
+  private String resolveCategory(String configured) throws CloudException {
+    if (configured != null) {
+      String wanted = stripSubcategoryPrefix(configured);
+      for (String category : categories) {
+        if (stripSubcategoryPrefix(category).equalsIgnoreCase(wanted)) {
+          return category;
+        }
+      }
+    }
+    throw new CloudException("Unknown category '" + configured + "', expected one of "
+        + categories);
+  }
+
+  // subcategories are listed as "- Name" and the server matches the category by that full string
+  private static String stripSubcategoryPrefix(String category) {
+    String trimmed = category.trim();
+    return trimmed.startsWith("- ") ? trimmed.substring(2).trim() : trimmed;
+  }
+
+  private static void checkResponse(String res) throws CloudException {
+    if (!SUCCESS.equals(res)) {
+      throw new CloudException(res);
+    }
+  }
+
+  static String projectName(URI uri) {
+    String path = uri.getPath();
+    String name = path.substring(path.lastIndexOf('/') + 1)
+        .replaceFirst("(?i)\\.diy$", "")
+        .replace('_', ' ')
+        .replaceAll("\\s+", " ")
+        .trim();
+    return name.length() > MAX_NAME_LENGTH ? name.substring(0, MAX_NAME_LENGTH).trim() : name;
+  }
+
+  static String description(String projectDescription, URI source) {
+    String sourceLine = "Source: " + source;
+    if (isBlank(projectDescription)) {
+      return sourceLine;
+    }
+    String text = projectDescription.trim();
+    int room = MAX_DESCRIPTION_LENGTH - sourceLine.length() - 2;
+    if (text.length() > room) {
+      text = text.substring(0, room).trim();
+    }
+    return text + "\n\n" + sourceLine;
+  }
+
+  static File localFile(File userDir, URI uri) throws IOException {
+    File root = userDir.getCanonicalFile();
+    File file = new File(new File(root, uri.getHost()), uri.getPath()).getCanonicalFile();
+    if (!file.toPath().startsWith(root.toPath())) {
+      throw new IOException("Refusing to store " + uri + " outside " + root);
+    }
+    return file;
+  }
+
+  private static String sha256(byte[] content) throws NoSuchAlgorithmException {
+    return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content));
+  }
+
+  private static boolean isBlank(String s) {
+    return s == null || s.isBlank();
+  }
+
+  private enum Outcome {
+    UNCHANGED, UPLOADED, UPDATED, FAILED
+  }
+
+  /**
+   * {@link Presenter#loadProjectFromFile(String)} reports a failed load only through the view, so
+   * the error is captured here to keep a stale project from being uploaded in its place.
+   */
+  private static class RecordingView extends DummyView {
+
+    private String error;
+
+    @Override
+    public void showMessage(String message, String title, int messageType) {
+      if (messageType == IView.ERROR_MESSAGE) {
+        error = message;
+      }
+    }
+
+    String takeError() {
+      String result = error;
+      error = null;
+      return result;
+    }
+  }
+}
@@ -0,0 +1,88 @@
+/*
+
+    DIY Layout Creator (DIYLC).
+    Copyright (c) 2009-2025 held jointly by the individual authors.
+
+    This file is part of DIYLC.
+
+    DIYLC is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    DIYLC is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with DIYLC.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+package org.diylc.swing.plugins.cloud.sync;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Finds links to .diy files on a web page and downloads them.
+ */
+public class PageScraper {
+
+  private static final Pattern DIY_LINK =
+      Pattern.compile("href\\s*=\\s*[\"']([^\"']+\\.diy)[\"']", Pattern.CASE_INSENSITIVE);
+  private static final Duration TIMEOUT = Duration.ofSeconds(60);
+  private static final String USER_AGENT = "DIYLC-CloudSync";
+
+  private final HttpClient client = HttpClient.newBuilder()
+      .followRedirects(HttpClient.Redirect.NORMAL)
+      .connectTimeout(TIMEOUT)
+      .build();
+
+  public List<URI> findDiyLinks(URI page) throws IOException, InterruptedException {
+    return extractDiyLinks(page, new String(download(page), StandardCharsets.UTF_8));
+  }
+
+  public byte[] download(URI uri) throws IOException, InterruptedException {
+    HttpRequest request = HttpRequest.newBuilder(uri)
+        .timeout(TIMEOUT)
+        .header("User-Agent", USER_AGENT)
+        .GET()
+        .build();
+    HttpResponse<byte[]> response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
+    if (response.statusCode() != 200) {
+      throw new IOException("HTTP " + response.statusCode() + " while downloading " + uri);
+    }
+    return response.body();
+  }
+
+  /**
+   * Returns absolute URIs of every distinct .diy link in the page, in page order. Links are taken
+   * as written in the markup, so entity-escaped ampersands are decoded and the raw spaces some
+   * pages use in file names are percent-encoded.
+   */
+  static List<URI> extractDiyLinks(URI base, String html) {
+    Set<URI> links = new LinkedHashSet<>();
+    Matcher matcher = DIY_LINK.matcher(html);
+    while (matcher.find()) {
+      String href = matcher.group(1).trim().replace("&amp;", "&").replace(" ", "%20");
+      try {
+        links.add(base.resolve(href).normalize());
+      } catch (IllegalArgumentException e) {
+        // not a well-formed URI even after encoding spaces; the link is unusable
+      }
+    }
+    return new ArrayList<>(links);
+  }
+}
@@ -0,0 +1,84 @@
+/*
+
+    DIY Layout Creator (DIYLC).
+    Copyright (c) 2009-2025 held jointly by the individual authors.
+
+    This file is part of DIYLC.
+
+    DIYLC is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    DIYLC is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with DIYLC.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+package org.diylc.swing.plugins.cloud.sync;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+/**
+ * Remembers, per source URL, which cloud project a file was uploaded as and the hash of the content
+ * that was uploaded, so that later runs only upload what is new or changed.
+ */
+public class SyncState {
+
+  private static final ObjectMapper MAPPER =
+      new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
+
+  private final File file;
+  private final Map<String, SyncedFile> filesByUrl = new LinkedHashMap<>();
+
+  private SyncState(File file) {
+    this.file = file;
+  }
+
+  public static SyncState load(File file) throws IOException {
+    SyncState state = new SyncState(file);
+    if (file.exists()) {
+      Snapshot snapshot = MAPPER.readValue(file, Snapshot.class);
+      for (SyncedFile synced : snapshot.files()) {
+        state.filesByUrl.put(synced.url(), synced);
+      }
+    }
+    return state;
+  }
+
+  public SyncedFile get(String url) {
+    return filesByUrl.get(url);
+  }
+
+  public boolean isClaimed(int projectId) {
+    return filesByUrl.values().stream().anyMatch(f -> f.projectId() == projectId);
+  }
+
+  /** Records the file and writes the state out immediately, so an interrupted run loses nothing. */
+  public void put(SyncedFile synced) throws IOException {
+    filesByUrl.put(synced.url(), synced);
+    file.getParentFile().mkdirs();
+    File temp = new File(file.getPath() + ".tmp");
+    MAPPER.writeValue(temp, new Snapshot(new ArrayList<>(filesByUrl.values())));
+    Files.move(temp.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING,
+        StandardCopyOption.ATOMIC_MOVE);
+  }
+
+  public record SyncedFile(String url, int projectId, String sha256, String syncedOn) {}
+
+  private record Snapshot(List<SyncedFile> files) {}
+}
@@ -0,0 +1,218 @@
+/*
+
+    DIY Layout Creator (DIYLC).
+    Copyright (c) 2009-2025 held jointly by the individual authors.
+
+    This file is part of DIYLC.
+
+    DIYLC is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    DIYLC is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with DIYLC.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+package org.diylc.swing.plugins.cloud.sync;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Proxy;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.diylc.appframework.miscutils.ConfigurationManager;
+import org.diylc.appframework.miscutils.InMemoryConfigurationManager;
+import org.diylc.common.DummyView;
+import org.diylc.plugins.cloud.model.IServiceAPI;
+import org.diylc.plugins.cloud.model.ProjectEntity;
+import org.diylc.presenter.Presenter;
+import org.diylc.swing.plugins.cloud.sync.CloudSyncConfig.PageConfig;
+import org.diylc.swing.plugins.cloud.sync.CloudSyncConfig.UserConfig;
+
+public class CloudSyncJobTests {
+
+  private static final URI PAGE = URI.create("https://example.com/files/Page/Page.htm");
+  private static final URI FILE = URI.create("https://example.com/files/Page/My_Amp.diy");
+
+  @Rule
+  public TemporaryFolder folder = new TemporaryFolder();
+
+  @Test
+  public void uploadsNewFilesOnceAndReplacesChangedOnes() throws Exception {
+    FakeCloud cloud = new FakeCloud();
+    FakeScraper scraper = new FakeScraper(emptyProject());
+
+    assertEquals(0, newJob(cloud, scraper).run());
+    assertEquals(List.of("search", "uploadProject My Amp", "search"), cloud.writesAndLookups);
+    assertEquals("Source: " + FILE, cloud.lastDescription);
+
+    cloud.writesAndLookups.clear();
+    assertEquals(0, newJob(cloud, scraper).run());
+    assertEquals(List.of(), cloud.writesAndLookups);
+
+    scraper.content = new String(scraper.content, StandardCharsets.UTF_8)
+        .replace("New Project", "Changed Project").getBytes(StandardCharsets.UTF_8);
+    assertEquals(0, newJob(cloud, scraper).run());
+    assertEquals(List.of("replaceProjectFiles 1"), cloud.writesAndLookups);
+    assertEquals(1, cloud.projects.size());
+  }
+
+  @Test
+  public void takesOverProjectAlreadyUploadedUnderSameName() throws Exception {
+    FakeCloud cloud = new FakeCloud();
+    cloud.add("My Amp");
+    cloud.add("Something Else");
+
+    assertEquals(0, newJob(cloud, new FakeScraper(emptyProject())).run());
+    assertEquals(List.of("search", "replaceProjectFiles 1"), cloud.writesAndLookups);
+  }
+
+  @Test
+  public void unreadableFileIsReportedAndNotUploaded() throws Exception {
+    FakeCloud cloud = new FakeCloud();
+    FakeScraper scraper = new FakeScraper("not a project".getBytes(StandardCharsets.UTF_8));
+
+    assertEquals(1, newJob(cloud, scraper).run());
+    assertEquals(List.of(), cloud.writesAndLookups);
+  }
+
+  @Test
+  public void extractsDistinctDiyLinksInPageOrder() {
+    String html = "<a href=\"https://example.com/files/Page/A_B.diy\">A_B.diy</a>"
+        + "<a href='Readme.htm'>readme</a>"
+        + "<a HREF=\"Sub/Bandmaster 5E7.DIY\">x</a>"
+        + "<a href=\"https://example.com/files/Page/A_B.diy\">again</a>";
+    List<URI> links = PageScraper.extractDiyLinks(PAGE, html);
+    assertEquals(List.of(URI.create("https://example.com/files/Page/A_B.diy"),
+        URI.create("https://example.com/files/Page/Sub/Bandmaster%205E7.DIY")), links);
+  }
+
+  @Test
+  public void derivesProjectNameFromFileName() {
+    assertEquals("Fender Bandmaster 5E7", CloudSyncJob.projectName(
+        URI.create("https://example.com/Fender_Bandmaster%205E7.diy")));
+    String longName = CloudSyncJob.projectName(URI.create("https://example.com/"
+        + "AA1164_Princeton_Reverb_Layout_Reverb_and_Tremolo_Deleted_Hoffman_Board.diy"));
+    assertEquals("AA1164 Princeton Reverb Layout Reverb and Tremolo", longName);
+  }
+
+  @Test
+  public void descriptionAlwaysCitesSource() {
+    URI source = URI.create("https://example.com/a.diy");
+    assertEquals("Source: https://example.com/a.diy", CloudSyncJob.description(null, source));
+    String description = CloudSyncJob.description("x".repeat(2000), source);
+    assertEquals(1024, description.length());
+    assertTrue(description.endsWith("\n\nSource: https://example.com/a.diy"));
+  }
+
+  @Test(expected = IOException.class)
+  public void refusesToStoreOutsideUserDirectory() throws IOException {
+    CloudSyncJob.localFile(new File("target/sync-test"),
+        URI.create("https://example.com/a/../../../../escape.diy"));
+  }
+
+  @Test
+  public void readsPagesAsBareUrlsOrObjects() throws IOException {
+    String json = "{\"users\":[{\"userId\":1,\"category\":\"Other\",\"pages\":["
+        + "\"https://example.com/a.htm\","
+        + "{\"url\":\"https://example.com/b.htm\",\"category\":\"Guitar Amplifiers\"}]}]}";
+    CloudSyncConfig config = new ObjectMapper().readValue(json, CloudSyncConfig.class);
+    CloudSyncConfig.UserConfig user = config.users().get(0);
+    assertEquals("Other", user.categoryFor(user.pages().get(0)));
+    assertEquals("Guitar Amplifiers", user.categoryFor(user.pages().get(1)));
+    assertEquals("", user.keywordsFor(user.pages().get(1)));
+    assertEquals(CloudSyncConfig.DEFAULT_SERVICE_URL, config.serviceUrlOrDefault());
+  }
+
+  private CloudSyncJob newJob(FakeCloud cloud, FakeScraper scraper) throws IOException {
+    UserConfig user = new UserConfig(7, "someone", "cron-job", "token", "Guitar Amplifiers", null,
+        List.of(PageConfig.of(PAGE.toString())));
+    CloudSyncConfig config = new CloudSyncConfig(null, null, List.of(user));
+    return new CloudSyncJob(config, new File(folder.getRoot(), "storage"), false, cloud.api(),
+        scraper);
+  }
+
+  private byte[] emptyProject() throws IOException {
+    ConfigurationManager.getInstance().initialize("diylc-test");
+    File file = folder.newFile("empty.diy");
+    new Presenter(new DummyView(), InMemoryConfigurationManager.getInstance())
+        .saveProjectToFile(file.getAbsolutePath(), false);
+    return Files.readAllBytes(file.toPath());
+  }
+
+  private static class FakeScraper extends PageScraper {
+
+    private byte[] content;
+
+    FakeScraper(byte[] content) {
+      this.content = content;
+    }
+
+    @Override
+    public List<URI> findDiyLinks(URI page) {
+      return List.of(FILE);
+    }
+
+    @Override
+    public byte[] download(URI uri) {
+      return content;
+    }
+  }
+
+  private static class FakeCloud {
+
+    private final List<ProjectEntity> projects = new ArrayList<>();
+    private final List<String> writesAndLookups = new ArrayList<>();
+    private String lastDescription;
+
+    void add(String name) {
+      ProjectEntity project = new ProjectEntity();
+      project.setId(projects.size() + 1);
+      project.setName(name);
+      projects.add(project);
+    }
+
+    IServiceAPI api() {
+      return (IServiceAPI) Proxy.newProxyInstance(getClass().getClassLoader(),
+          new Class<?>[] {IServiceAPI.class}, (proxy, method, args) -> {
+            switch (method.getName()) {
+              case "getCategories":
+                return new ArrayList<>(List.of("Other", "- Guitar Amplifiers"));
+              case "search":
+                writesAndLookups.add("search");
+                return new ArrayList<>(projects);
+              case "uploadProject":
+                assertEquals("- Guitar Amplifiers", args[4]);
+                lastDescription = (String) args[5];
+                add((String) args[3]);
+                writesAndLookups.add("uploadProject " + args[3]);
+                return "Success";
+              case "replaceProjectFiles":
+                writesAndLookups.add("replaceProjectFiles " + args[6]);
+                return "Success";
+              default:
+                throw new UnsupportedOperationException(method.getName());
+            }
+          });
+    }
+  }
+}