diff --git a/.github/scripts/find-modules-with-unit-tests.sh b/.github/scripts/find-modules-with-unit-tests.sh index b6900852ccb..4723885f342 100755 --- a/.github/scripts/find-modules-with-unit-tests.sh +++ b/.github/scripts/find-modules-with-unit-tests.sh @@ -1,7 +1,7 @@ #!/bin/bash -e find . -path '**/src/test/java' -type d \ - | grep -v -E '\./(docs|distribution|misc|operator|((.+/)?tests)|testsuite|test-framework|quarkus)/' \ + | grep -v -E '\./(docs|distribution|operator|((.+/)?tests)|testsuite|test-framework|quarkus)/' \ | sed 's|/src/test/java||' \ | sed 's|./||' \ | sort \ diff --git a/misc/db-compatibility-verifier/README.md b/misc/db-compatibility-verifier/README.md new file mode 100644 index 00000000000..e8655174093 --- /dev/null +++ b/misc/db-compatibility-verifier/README.md @@ -0,0 +1,87 @@ +# Database Compatibility Verifier Maven Plugin + +## Overview + +This Maven plugin is used to verify the database compatibility of Keycloak. It ensures that all database schema changes +(ChangeSets) are explicitly marked as either supported or unsupported by the rolling upgrades feature. + +## Goals + +The plugin provides the following goals: + +* `db-compatibility-verifier:snapshot` +* `db-compatibility-verifier:verify` +* `db-compatibility-verifier:supported` +* `db-compatibility-verifier:unsupported` + +## Usage + +### `snapshot` - Creates a snapshot of the current database ChangeSets. + +This goal is used to create an initial snapshot of the database ChangeSets. It creates a supported and unsupported JSON +file, specified via the `db.verify.supportedFile` and `db.verify.unsupportedFile` property, respectively. + +```bash +mvn org.keycloak:db-compatibility-verifier-maven-plugin:999.0.0-SNAPSHOT:snapshot \ + -Ddb.verify.supportedFile= \ + -Ddb.verify.unsupportedFile= +``` + +The `supportedFile` will be created with a record of all known ChangeSets and the `unsupportedFile` will be initialized +as an empty JSON array. + +### `verify` - Verifies that all detected ChangeSets recorded in either the supported or unsupported JSON files. + +```bash +mvn org.keycloak:db-compatibility-verifier-maven-plugin:999.0.0-SNAPSHOT:verify \ + -Ddb.verify.supportedFile= \ + -Ddb.verify.unsupportedFile= +``` + +### `supported` - Adds one or all missing ChangeSets to the supported JSON file + +This goal is used to mark a ChangeSet as supported for rolling upgrades. + +To mark a single ChangeSet as supported: + +```bash +mvn org.keycloak:db-compatibility-verifier-maven-plugin:999.0.0-SNAPSHOT:supported \ + -Ddb.verify.supportedFile= \ + -Ddb.verify.unsupportedFile= \ + -Ddb.verify.changset.id= \ + -Ddb.verify.changset.author= \ + -Ddb.verify.changset.filename= +``` + +To mark all missing ChangeSets as supported: + +```bash +mvn org.keycloak:db-compatibility-verifier-maven-plugin:999.0.0-SNAPSHOT:supported \ + -Ddb.verify.supportedFile= \ + -Ddb.verify.unsupportedFile= \ + -Ddb.verify.changset.addAll=true +``` + +### `unsupported` - Adds one or all missing ChangeSets to the unsupported JSON file + +This goal is used to mark a ChangeSet as unsupported for rolling upgrades. + +To mark a single ChangeSet as unsupported: + +```bash +mvn org.keycloak:db-compatibility-verifier-maven-plugin:999.0.0-SNAPSHOT:unsupported \ + -Ddb.verify.supportedFile= \ + -Ddb.verify.unsupportedFile= \ + -Ddb.verify.changset.id= \ + -Ddb.verify.changset.author= \ + -Ddb.verify.changset.filename= +``` + +To mark all missing ChangeSets as unsupported: + +```bash +mvn org.keycloak:db-compatibility-verifier-maven-plugin:999.0.0-SNAPSHOT:unsupported \ + -Ddb.verify.supportedFile= \ + -Ddb.verify.unsupportedFile= \ + -Ddb.verify.changset.addAll=true +``` diff --git a/misc/db-compatibility-verifier/pom.xml b/misc/db-compatibility-verifier/pom.xml new file mode 100644 index 00000000000..0514f6bf6c4 --- /dev/null +++ b/misc/db-compatibility-verifier/pom.xml @@ -0,0 +1,76 @@ + + + 4.0.0 + + + org.keycloak + keycloak-parent + 999.0.0-SNAPSHOT + ../../pom.xml + + + db-compatibility-verifier-maven-plugin + + Database Compatibilility Verifier + Database Compatibility Verifier + + maven-plugin + + + 17 + 17 + UTF-8 + + + + + org.apache.maven + maven-plugin-api + ${maven.version} + provided + + + org.apache.maven.plugin-tools + maven-plugin-annotations + ${maven.plugin-tools.version} + provided + + + org.apache.maven + maven-core + ${maven.version} + provided + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + org.junit.jupiter + junit-jupiter-api + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/AbstractChangeSetMojo.java b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/AbstractChangeSetMojo.java new file mode 100644 index 00000000000..411b8ace2a6 --- /dev/null +++ b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/AbstractChangeSetMojo.java @@ -0,0 +1,83 @@ +package org.keycloak.db.compatibility.verifier; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Parameter; + +abstract class AbstractChangeSetMojo extends AbstractMojo { + @Parameter(property = "db.verify.changeset.all", defaultValue = "false") + boolean addAll; + + @Parameter(property = "db.verify.changeset.id") + String id; + + @Parameter(property = "db.verify.changeset.author") + String author; + + @Parameter(property = "db.verify.changeset.filename") + String filename; + + + void checkFileExist(String ref, File file) throws MojoExecutionException { + if (!file.exists()) { + throw new MojoExecutionException("%s file does not exist".formatted(ref)); + } + } + + void checkUnknownChangeSet(Set knownChangeSets, ChangeSet changeSet) throws MojoExecutionException { + if (!knownChangeSets.contains(changeSet)) { + throw new MojoExecutionException("Unknown ChangeSet: " + changeSet); + } + } + + protected void checkValidChangeSetId(String id, String author, String filename) throws MojoExecutionException { + if (id == null || id.isBlank()) { + throw new MojoExecutionException("ChangeSet id not set"); + } + if (author == null || author.isBlank()) { + throw new MojoExecutionException("ChangeSet author not set"); + } + if (filename == null || filename.isBlank()) { + throw new MojoExecutionException("ChangeSet filename not set"); + } + } + + void addAll(ClassLoader classLoader, File dest, File exclusions) throws IOException { + // Discover all known ChangeSets + ChangeLogXMLParser xmlParser = new ChangeLogXMLParser(classLoader); + Set knownChangeSets = xmlParser.discoverAllChangeSets(); + + // Load changes to exclude and remove them from the known changesets + Set excludedChanges = objectMapper.readValue(exclusions, new TypeReference<>() {}); + knownChangeSets.removeAll(excludedChanges); + + // Overwrite all content in the destination file + objectMapper.writeValue(dest, knownChangeSets); + } + + void addIndividual(ClassLoader classLoader, ChangeSet changeSet, File dest, File alternate) throws IOException, MojoExecutionException { + // Discover all known ChangeSets + ChangeLogXMLParser xmlParser = new ChangeLogXMLParser(classLoader); + Set knownChangeSets = xmlParser.discoverAllChangeSets(); + + // It should not be possible to add an unknown changeset + checkUnknownChangeSet(knownChangeSets, changeSet); + + Set alternateChangeSets = objectMapper.readValue(alternate, new TypeReference<>() {}); + if (alternateChangeSets.contains(changeSet)) { + throw new MojoExecutionException("ChangeSet already defined in the %s file".formatted(alternate.getName())); + } + + List destChanges = objectMapper.readValue(dest, new TypeReference<>() {}); + if (!destChanges.contains(changeSet)) { + // If the ChangeSet is not already known, append to the end of the JSON array and overwrite the existing file + destChanges.add(changeSet); + objectMapper.writeValue(dest, destChanges); + } + } +} diff --git a/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/AbstractMojo.java b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/AbstractMojo.java new file mode 100644 index 00000000000..6077e68f18b --- /dev/null +++ b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/AbstractMojo.java @@ -0,0 +1,39 @@ +package org.keycloak.db.compatibility.verifier; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.List; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.apache.maven.artifact.DependencyResolutionRequiredException; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; + +abstract class AbstractMojo extends org.apache.maven.plugin.AbstractMojo { + + final ObjectMapper objectMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); + + @Parameter(defaultValue = "${project}", readonly = true) + protected MavenProject project; + + @Parameter(property = "db.verify.supportedFile", required = true) + protected String supportedFile; + + @Parameter(property = "db.verify.unsupportedFile", required = true) + protected String unsupportedFile; + + @Parameter(property = "db.verify.skip", defaultValue = "false") + protected boolean skip; + + ClassLoader classLoader() throws DependencyResolutionRequiredException, MalformedURLException { + List elements = project.getRuntimeClasspathElements(); + URL[] urls = new URL[elements.size()]; + for (int i = 0; i < elements.size(); i++) { + urls[i] = new File(elements.get(i)).toURI().toURL(); + } + return new URLClassLoader(urls, null); + } +} diff --git a/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeLogXMLParser.java b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeLogXMLParser.java new file mode 100644 index 00000000000..49e9ba7723e --- /dev/null +++ b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeLogXMLParser.java @@ -0,0 +1,135 @@ +package org.keycloak.db.compatibility.verifier; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +record ChangeLogXMLParser(ClassLoader classLoader) { + + static final String RESOURCE_DIR = "META-INF"; + + Set discoverAllChangeSets() throws IOException { + var changeSets = changeSetXmlFiles() + .map(this::extractChangeSets) + .flatMap(List::stream) + .toList(); + + Set uniqueSets = new HashSet<>(changeSets.size()); + ChangeSet duplicate = changeSets.stream() + .filter(item -> !uniqueSets.add(item)) + .findFirst() + .orElse(null); + if (duplicate != null) { + throw new IllegalStateException("Duplicate ChangeSet detected: " + duplicate); + } + return uniqueSets; + } + + List extractChangeSets(String filename) { + XMLInputFactory factory = XMLInputFactory.newInstance(); + // Security: Disable DTDs to prevent XML External Entity (XXE) attacks + factory.setProperty(XMLInputFactory.SUPPORT_DTD, false); + factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + List ids = new ArrayList<>(); + + try (InputStream is = classLoader.getResourceAsStream(filename)) { + XMLStreamReader reader = factory.createXMLStreamReader(is); + while (reader.hasNext()) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String tagName = reader.getLocalName(); + + // 1. Handle Root Element + if (tagName.equals("databaseChangeLog")) { + continue; + } + + // 2. Process changeSet + if (tagName.equals("changeSet")) { + String id = reader.getAttributeValue(null, "id"); + String author = reader.getAttributeValue(null, "author"); + ids.add(new ChangeSet(id, author, filename)); + + // Skip all child elements until we find the closing + skipUnknownElement(reader); + continue; + } + // 3. Skip all other elements + skipUnknownElement(reader); + } + } + return ids; + } catch (IOException | XMLStreamException e) { + throw new IllegalStateException(e); + } + } + + // Detect all jpa-changelog*.xml files on the classpath + private Stream changeSetXmlFiles() throws IOException { + List fileNames = new ArrayList<>(); + Enumeration en = classLoader.getResources(RESOURCE_DIR); + + while (en.hasMoreElements()) { + URI uri; + try { + uri = en.nextElement().toURI(); + } catch (URISyntaxException e) { + // Should never happen + throw new IllegalStateException(e); + } + + if (uri.getScheme().equals("jar")) { + // Handle JAR resources + try (FileSystem fs = FileSystems.newFileSystem(uri, Collections.emptyMap())) { + Path path = fs.getPath(RESOURCE_DIR); + fileNames.addAll(listFromPath(path)); + } + } else { + // Handle local file system (IDE) + fileNames.addAll(listFromPath(Paths.get(uri))); + } + } + return fileNames.stream() + .filter(s -> s.startsWith("jpa-changelog") && s.endsWith(".xml")) + .map(s -> "%s/%s".formatted(RESOURCE_DIR, s)); + } + + private List listFromPath(Path path) throws IOException { + try (Stream walk = Files.walk(path, 1)) { + return walk.filter(Files::isRegularFile) + .map(p -> p.getFileName().toString()) + .toList(); + } + } + + private static void skipUnknownElement(XMLStreamReader reader) throws XMLStreamException { + int level = 1; + while (level > 0 && reader.hasNext()) { + int event = reader.next(); + if (event == XMLStreamConstants.START_ELEMENT) { + level++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + level--; + } + } + } +} diff --git a/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeSet.java b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeSet.java new file mode 100644 index 00000000000..6e4c5a00dbe --- /dev/null +++ b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeSet.java @@ -0,0 +1,4 @@ +package org.keycloak.db.compatibility.verifier; + +record ChangeSet(String id, String author, String filename) { +} diff --git a/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeSetSupportedMojo.java b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeSetSupportedMojo.java new file mode 100644 index 00000000000..58a980c1904 --- /dev/null +++ b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeSetSupportedMojo.java @@ -0,0 +1,37 @@ +package org.keycloak.db.compatibility.verifier; + +import java.io.File; + +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; + +@Mojo(name = "supported") +public class ChangeSetSupportedMojo extends AbstractChangeSetMojo { + + @Override + public void execute() throws MojoExecutionException { + if (skip) { + getLog().info("Skipping execution"); + return; + } + + File root = project.getBasedir(); + File sFile = new File(root, supportedFile); + File uFile = new File(root, unsupportedFile); + checkFileExist("supported", sFile); + checkFileExist("unsupported", uFile); + + try { + if (addAll) { + addAll(classLoader(), sFile, uFile); + } else { + checkValidChangeSetId(id, author, filename); + ChangeSet changeSet = new ChangeSet(id, author, filename); + addIndividual(classLoader(), changeSet, sFile, uFile); + } + } catch (Exception e) { + throw new MojoExecutionException("Error adding ChangeSet to supported file", e); + } + } + +} diff --git a/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeSetUnsupportedMojo.java b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeSetUnsupportedMojo.java new file mode 100644 index 00000000000..eeae2a2fc84 --- /dev/null +++ b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/ChangeSetUnsupportedMojo.java @@ -0,0 +1,36 @@ +package org.keycloak.db.compatibility.verifier; + +import java.io.File; + +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; + +@Mojo(name = "unsupported") +public class ChangeSetUnsupportedMojo extends AbstractChangeSetMojo { + + @Override + public void execute() throws MojoExecutionException { + if (skip) { + getLog().info("Skipping execution"); + return; + } + + File root = project.getBasedir(); + File sFile = new File(root, supportedFile); + File uFile = new File(root, unsupportedFile); + checkFileExist("supported", sFile); + checkFileExist("unsupported", uFile); + + try { + if (addAll) { + addAll(classLoader(), uFile, sFile); + } else { + checkValidChangeSetId(id, author, filename); + ChangeSet changeSet = new ChangeSet(id, author, filename); + addIndividual(classLoader(), changeSet, uFile, sFile); + } + } catch (Exception e) { + throw new MojoExecutionException("Error adding ChangeSet to unsupported file", e); + } + } +} diff --git a/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/CreateSnapshotMojo.java b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/CreateSnapshotMojo.java new file mode 100644 index 00000000000..a401aa59929 --- /dev/null +++ b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/CreateSnapshotMojo.java @@ -0,0 +1,43 @@ +package org.keycloak.db.compatibility.verifier; + +import java.io.File; +import java.io.IOException; +import java.util.Set; + +import com.fasterxml.jackson.databind.SerializationFeature; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; + +@Mojo(name = "snapshot") +public class CreateSnapshotMojo extends AbstractMojo { + + @Override + public void execute() throws MojoExecutionException { + if (skip) { + getLog().info("Skipping execution"); + return; + } + + try { + File root = project.getBasedir(); + File sFile = new File(root, supportedFile); + File uFile = new File(root, unsupportedFile); + + ClassLoader classLoader = classLoader(); + createSnapshot(classLoader, sFile, uFile); + } catch (Exception e) { + throw new MojoExecutionException("Error creating ChangeSet snapshot", e); + } + } + + void createSnapshot(ClassLoader classLoader, File sFile, File uFile) throws IOException { + // Write all known ChangeSet defined in the jpa-changelog*.xml files to the supported file + ChangeLogXMLParser xmlParser = new ChangeLogXMLParser(classLoader); + Set changeSets = xmlParser.discoverAllChangeSets(); + objectMapper.enable(SerializationFeature.INDENT_OUTPUT); + objectMapper.writeValue(sFile, changeSets); + + // Create an empty JSON array in the unsupported file + objectMapper.writeValue(uFile, Set.of()); + } +} diff --git a/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/VerifyCompatibilityMojo.java b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/VerifyCompatibilityMojo.java new file mode 100644 index 00000000000..13156e65282 --- /dev/null +++ b/misc/db-compatibility-verifier/src/main/java/org/keycloak/db/compatibility/verifier/VerifyCompatibilityMojo.java @@ -0,0 +1,90 @@ +package org.keycloak.db.compatibility.verifier; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.Mojo; + +@Mojo(name = "verify") +public class VerifyCompatibilityMojo extends AbstractMojo { + + @Override + public void execute() throws MojoExecutionException { + if (skip) { + getLog().info("Skipping execution"); + return; + } + + try { + File root = project.getBasedir(); + File sFile = new File(root, supportedFile); + File uFile = new File(root, unsupportedFile); + verifyCompatibility(classLoader(), sFile, uFile); + } catch (Exception e) { + throw new MojoExecutionException("Error loading project resources", e); + } + } + + void verifyCompatibility(ClassLoader classLoader, File sFile, File uFile) throws IOException, MojoExecutionException { + if (!sFile.exists() && !uFile.exists()) { + getLog().info("No JSON ChangeSet files exist to verify"); + return; + } + + // Parse JSON files to determine all committed ChangeSets + List sChanges = objectMapper.readValue(sFile, new TypeReference<>() {}); + List uChanges = objectMapper.readValue(uFile, new TypeReference<>() {}); + Set recordedChanges = Stream.of(sChanges, uChanges) + .flatMap(List::stream) + .collect(Collectors.toSet()); + + if (recordedChanges.isEmpty()) { + getLog().info("No supported or unsupported ChangeSet exist in specified files"); + return; + } + + checkIntersection(sChanges, uChanges); + + // Parse all ChangeSets currently defined in the jpa-changegetLog() files + ChangeLogXMLParser xmlParser = new ChangeLogXMLParser(classLoader); + Set currentChanges = xmlParser.discoverAllChangeSets(); + checkMissingChangeSet(currentChanges, recordedChanges, sFile, uFile); + } + + void checkIntersection(List sChanges, List uChanges) throws MojoExecutionException { + Set intersection = new HashSet<>(sChanges); + intersection.retainAll(uChanges); + if (!intersection.isEmpty()) { + getLog().error("The following ChangeSets should be defined in either the supported or unsupported file, they cannot appear in both:"); + intersection.forEach(change -> getLog().error("\t\t" + change.toString())); + getLog().error("The offending ChangeSets should be removed from one of the files"); + throw new MojoExecutionException("One or more ChangeSet definitions exist in both the supported and unsupported file"); + } + } + + void checkMissingChangeSet(Set currentChanges, Set recordedChanges, File sFile, File uFile) throws MojoExecutionException { + if (recordedChanges.equals(currentChanges)) { + getLog().info("All ChangeSets in the module recorded as expected in the supported and unsupported files"); + } else { + getLog().error("The recorded ChangeSet JSON files differ from the current repository state"); + getLog().error("The following ChangeSets should be defined in either the supported '%s' or unsupported '%s' file:".formatted(sFile.toString(), uFile.toString())); + currentChanges.removeAll(recordedChanges); + currentChanges.forEach(change -> getLog().error("\t\t" + change.toString())); + getLog().error("You must determine whether the ChangeSet(s) is compatible with rolling upgrades or not"); + getLog().error("A ChangeSet that requires locking preventing other cluster members accessing the database or makes schema changes that breaks functionality in earlier Keycloak versions is NOT compatible with rolling upgrades"); + getLog().error("Rolling upgrade compatibility must be verified against all supported database vendors before the supported file is updated"); + getLog().error("If the change IS compatible, then it should be committed to the repository in the supported file: '%s'".formatted(sFile.toString())); + getLog().error("If the change IS NOT compatible, then it should be committed to the repository in the unsupported file: '%s'".formatted(sFile.toString())); + getLog().error("Adding a ChangeSet to the unsupported file ensures that a rolling upgrade is not attempted when upgrading to the first patch version containing the change"); + getLog().error("ChangeSets can be added to the supported or unsupported files using the org.keycloak:db-compatibility-verifier-maven-plugin. See the module README for usage instructions"); + throw new MojoExecutionException("One or more ChangeSet definitions are missing from the supported or unsupported files"); + } + } +} diff --git a/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/AbstractMojoTest.java b/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/AbstractMojoTest.java new file mode 100644 index 00000000000..7767371c7dc --- /dev/null +++ b/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/AbstractMojoTest.java @@ -0,0 +1,44 @@ +package org.keycloak.db.compatibility.verifier; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +abstract class AbstractMojoTest { + protected Path testDir; + protected File supportedFile; + protected File unsupportedFile; + + @BeforeEach + void init() throws IOException { + testDir = Files.createTempDirectory(ChangeSetSupportedMojoTest.class.getSimpleName()); + supportedFile = testDir.resolve("supported.json").toFile(); + unsupportedFile = testDir.resolve("unsupported.json").toFile(); + } + + @AfterEach + void cleanup() throws IOException { + if (Files.exists(testDir)) { + Files.walkFileTree(testDir, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + } +} diff --git a/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/ChangeSetSupportedMojoTest.java b/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/ChangeSetSupportedMojoTest.java new file mode 100644 index 00000000000..c80184ee1c0 --- /dev/null +++ b/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/ChangeSetSupportedMojoTest.java @@ -0,0 +1,125 @@ +package org.keycloak.db.compatibility.verifier; + +import java.util.List; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.maven.plugin.MojoExecutionException; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ChangeSetSupportedMojoTest extends AbstractMojoTest { + + @Test + void testAddAll() throws Exception { + var classLoader = ChangeSetSupportedMojoTest.class.getClassLoader(); + var mojo = new ChangeSetSupportedMojo(); + var mapper = new ObjectMapper(); + + // Create unsupported file with a single ChangeSet + List unsupportedChanges = new ChangeLogXMLParser(classLoader).extractChangeSets("META-INF/jpa-changelog-2.xml"); + assertEquals(1, unsupportedChanges.size()); + mapper.writeValue(unsupportedFile, unsupportedChanges); + + // Execute add all and expect all ChangeSets from jpa-changelog-1.xml to be present + assertTrue(supportedFile.createNewFile()); + mojo.addAll(classLoader, supportedFile, unsupportedFile); + + List supportedChanges = mapper.readValue(supportedFile, new TypeReference<>() {}); + assertEquals(1, supportedChanges.size()); + + ChangeSet sChange = supportedChanges.get(0); + assertEquals("test", sChange.id()); + assertEquals("keycloak", sChange.author()); + assertEquals("META-INF/jpa-changelog-1.xml", sChange.filename()); + } + + @Test + void testAddIndividual() throws Exception { + var classLoader = ChangeSetSupportedMojoTest.class.getClassLoader(); + var changeLogParser = new ChangeLogXMLParser(classLoader); + var mojo = new ChangeSetSupportedMojo(); + var mapper = new ObjectMapper(); + + assertTrue(supportedFile.createNewFile()); + assertTrue(unsupportedFile.createNewFile()); + mapper.writeValue(supportedFile, List.of()); + mapper.writeValue(unsupportedFile, List.of()); + + // Test ChangeSet is added to supported file as expected + ChangeSet changeSet = changeLogParser.extractChangeSets("META-INF/jpa-changelog-1.xml").get(0); + mojo.addIndividual(classLoader, changeSet, supportedFile, unsupportedFile); + + List supportedChanges = mapper.readValue(supportedFile, new TypeReference<>() {}); + assertEquals(1, supportedChanges.size()); + ChangeSet sChange = supportedChanges.get(0); + assertEquals(changeSet.id(), sChange.id()); + assertEquals(changeSet.author(), sChange.author()); + assertEquals(changeSet.filename(), sChange.filename()); + + // Test subsequent ChangeSets are added to already populated supported file + changeSet = changeLogParser.extractChangeSets("META-INF/jpa-changelog-2.xml").get(0); + mojo.addIndividual(classLoader, changeSet, supportedFile, unsupportedFile); + + supportedChanges = mapper.readValue(supportedFile, new TypeReference<>() {}); + assertEquals(2, supportedChanges.size()); + + sChange = supportedChanges.get(1); + assertEquals(changeSet.id(), sChange.id()); + assertEquals(changeSet.author(), sChange.author()); + assertEquals(changeSet.filename(), sChange.filename()); + + // Test ChangeSet already exists handled gracefully + mojo.addIndividual(classLoader, changeSet, supportedFile, unsupportedFile); + + supportedChanges = mapper.readValue(supportedFile, new TypeReference<>() {}); + assertEquals(2, supportedChanges.size()); + } + + @Test + void testChangeAlreadyUnsupported() throws Exception { + var classLoader = ChangeSetSupportedMojoTest.class.getClassLoader(); + var mojo = new ChangeSetSupportedMojo(); + var mapper = new ObjectMapper(); + + assertTrue(supportedFile.createNewFile()); + assertTrue(unsupportedFile.createNewFile()); + mapper.writeValue(supportedFile, List.of()); + + // Create unsupported file with a single ChangeSet + List unsupportedChanges = new ChangeLogXMLParser(classLoader).extractChangeSets("META-INF/jpa-changelog-1.xml"); + assertEquals(1, unsupportedChanges.size()); + + ChangeSet changeSet = unsupportedChanges.get(0); + mapper.writeValue(unsupportedFile, unsupportedChanges); + + Exception e = assertThrows( + MojoExecutionException.class, + () -> mojo.addIndividual(classLoader, changeSet, supportedFile, unsupportedFile) + ); + + assertEquals("ChangeSet already defined in the %s file".formatted(unsupportedFile.getName()), e.getMessage()); + } + + @Test + void testAddUnknownChangeSet() throws Exception { + var classLoader = ChangeSetSupportedMojoTest.class.getClassLoader(); + var mojo = new ChangeSetSupportedMojo(); + var mapper = new ObjectMapper(); + + assertTrue(supportedFile.createNewFile()); + assertTrue(unsupportedFile.createNewFile()); + mapper.writeValue(supportedFile, List.of()); + ChangeSet unknown = new ChangeSet("asf", "asfgasg", "afasgfas"); + + Exception e = assertThrows( + MojoExecutionException.class, + () -> mojo.addIndividual(classLoader, unknown, supportedFile, unsupportedFile) + ); + + assertEquals("Unknown ChangeSet: " + unknown, e.getMessage()); + } +} diff --git a/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/ChangeSetUnsupportedMojoTest.java b/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/ChangeSetUnsupportedMojoTest.java new file mode 100644 index 00000000000..7612393969b --- /dev/null +++ b/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/ChangeSetUnsupportedMojoTest.java @@ -0,0 +1,125 @@ +package org.keycloak.db.compatibility.verifier; + +import java.util.List; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.maven.plugin.MojoExecutionException; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ChangeSetUnsupportedMojoTest extends AbstractMojoTest { + + @Test + void testAddAll() throws Exception { + var classLoader = ChangeSetUnsupportedMojoTest.class.getClassLoader(); + var mojo = new ChangeSetUnsupportedMojo(); + var mapper = new ObjectMapper(); + + // Create supported file with a single ChangeSet + List supportedChanges = new ChangeLogXMLParser(classLoader).extractChangeSets("META-INF/jpa-changelog-2.xml"); + assertEquals(1, supportedChanges.size()); + mapper.writeValue(unsupportedFile, supportedChanges); + + // Execute add all and expect all ChangeSets from jpa-changelog-1.xml to be present + assertTrue(supportedFile.createNewFile()); + mojo.addAll(classLoader, supportedFile, unsupportedFile); + + List unsupportedChanges = mapper.readValue(supportedFile, new TypeReference<>() {}); + assertEquals(1, unsupportedChanges.size()); + + ChangeSet sChange = unsupportedChanges.get(0); + assertEquals("test", sChange.id()); + assertEquals("keycloak", sChange.author()); + assertEquals("META-INF/jpa-changelog-1.xml", sChange.filename()); + } + + @Test + void testAddIndividual() throws Exception { + var classLoader = ChangeSetUnsupportedMojoTest.class.getClassLoader(); + var changeLogParser = new ChangeLogXMLParser(classLoader); + var mojo = new ChangeSetUnsupportedMojo(); + var mapper = new ObjectMapper(); + + assertTrue(supportedFile.createNewFile()); + assertTrue(unsupportedFile.createNewFile()); + mapper.writeValue(supportedFile, List.of()); + mapper.writeValue(unsupportedFile, List.of()); + + // Test ChangeSet is added to unsupported file as expected + ChangeSet changeSet = changeLogParser.extractChangeSets("META-INF/jpa-changelog-1.xml").get(0); + mojo.addIndividual(classLoader, changeSet, unsupportedFile, supportedFile); + + List unsupportedChanges = mapper.readValue(unsupportedFile, new TypeReference<>() {}); + assertEquals(1, unsupportedChanges.size()); + ChangeSet sChange = unsupportedChanges.get(0); + assertEquals(changeSet.id(), sChange.id()); + assertEquals(changeSet.author(), sChange.author()); + assertEquals(changeSet.filename(), sChange.filename()); + + // Test subsequent ChangeSets are added to already populated supported file + changeSet = changeLogParser.extractChangeSets("META-INF/jpa-changelog-2.xml").get(0); + mojo.addIndividual(classLoader, changeSet, unsupportedFile, supportedFile); + + unsupportedChanges = mapper.readValue(unsupportedFile, new TypeReference<>() {}); + assertEquals(2, unsupportedChanges.size()); + + sChange = unsupportedChanges.get(1); + assertEquals(changeSet.id(), sChange.id()); + assertEquals(changeSet.author(), sChange.author()); + assertEquals(changeSet.filename(), sChange.filename()); + + // Test ChangeSet already exists handled gracefully + mojo.addIndividual(classLoader, changeSet, unsupportedFile, supportedFile); + + unsupportedChanges = mapper.readValue(unsupportedFile, new TypeReference<>() {}); + assertEquals(2, unsupportedChanges.size()); + } + + @Test + void testChangeAlreadySupported() throws Exception { + var classLoader = ChangeSetUnsupportedMojoTest.class.getClassLoader(); + var mojo = new ChangeSetUnsupportedMojo(); + var mapper = new ObjectMapper(); + + assertTrue(supportedFile.createNewFile()); + assertTrue(unsupportedFile.createNewFile()); + mapper.writeValue(unsupportedFile, List.of()); + + // Create supported file with a single ChangeSet + List unsupportedChanges = new ChangeLogXMLParser(classLoader).extractChangeSets("META-INF/jpa-changelog-1.xml"); + assertEquals(1, unsupportedChanges.size()); + + ChangeSet changeSet = unsupportedChanges.get(0); + mapper.writeValue(supportedFile, unsupportedChanges); + + Exception e = assertThrows( + MojoExecutionException.class, + () -> mojo.addIndividual(classLoader, changeSet, unsupportedFile, supportedFile) + ); + + assertEquals("ChangeSet already defined in the %s file".formatted(supportedFile.getName()), e.getMessage()); + } + + @Test + void testAddUnknownChangeSet() throws Exception { + var classLoader = ChangeSetSupportedMojoTest.class.getClassLoader(); + var mojo = new ChangeSetSupportedMojo(); + var mapper = new ObjectMapper(); + + assertTrue(supportedFile.createNewFile()); + assertTrue(unsupportedFile.createNewFile()); + mapper.writeValue(unsupportedFile, List.of()); + ChangeSet unknown = new ChangeSet("asf", "asfgasg", "afasgfas"); + + Exception e = assertThrows( + MojoExecutionException.class, + () -> mojo.addIndividual(classLoader, unknown, unsupportedFile, supportedFile) + ); + + assertEquals("Unknown ChangeSet: " + unknown, e.getMessage()); + } +} diff --git a/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/CreateSnapshotMojoTest.java b/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/CreateSnapshotMojoTest.java new file mode 100644 index 00000000000..9150464343f --- /dev/null +++ b/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/CreateSnapshotMojoTest.java @@ -0,0 +1,30 @@ +package org.keycloak.db.compatibility.verifier; + +import java.util.List; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class CreateSnapshotMojoTest extends AbstractMojoTest { + + @Test + void testSnapshotFilesCreated() throws Exception { + var classLoader = CreateSnapshotMojoTest.class.getClassLoader(); + var mojo = new CreateSnapshotMojo(); + mojo.createSnapshot(classLoader, supportedFile, unsupportedFile); + + assertTrue(supportedFile.exists()); + assertTrue(unsupportedFile.exists()); + + var mapper = new ObjectMapper(); + List supportedChanges = mapper.readValue(supportedFile, new TypeReference<>() {}); + assertEquals(2, supportedChanges.size()); + + List unsupportedChanges = mapper.readValue(unsupportedFile, new TypeReference<>() {}); + assertEquals(0, unsupportedChanges.size()); + } +} diff --git a/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/VerifyCompatibilityMojoTest.java b/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/VerifyCompatibilityMojoTest.java new file mode 100644 index 00000000000..4c49bf39ddc --- /dev/null +++ b/misc/db-compatibility-verifier/src/test/java/org/keycloak/db/compatibility/verifier/VerifyCompatibilityMojoTest.java @@ -0,0 +1,75 @@ +package org.keycloak.db.compatibility.verifier; + +import java.io.File; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.maven.plugin.MojoExecutionException; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class VerifyCompatibilityMojoTest { + + final ClassLoader classLoader = VerifyCompatibilityMojoTest.class.getClassLoader(); + + @Test + void testChangeSetFilesDoNotExist() { + var mojo = new VerifyCompatibilityMojo(); + File noneExistingFile = new File("noneExistingFile"); + assertFalse(noneExistingFile.exists()); + + assertDoesNotThrow(() -> mojo.verifyCompatibility(classLoader, noneExistingFile, noneExistingFile)); + } + + @Test + void testEmptyChangeSetFiles() { + var mojo = new VerifyCompatibilityMojo(); + File emptyJson = new File(classLoader.getResource("META-INF/empty-array.json").getFile()); + + assertDoesNotThrow(() -> mojo.verifyCompatibility(classLoader, emptyJson, emptyJson)); + } + + @Test + void testChangeSetIncludedInSupportedAndUnsupportedFiles() { + var mojo = new VerifyCompatibilityMojo(); + var changeSet = new ChangeSet("1", "keycloak", "example.xml"); + + Exception e = assertThrows( + MojoExecutionException.class, + () -> mojo.checkIntersection(List.of(changeSet), List.of(changeSet)) + ); + assertEquals("One or more ChangeSet definitions exist in both the supported and unsupported file", e.getMessage()); + } + + @Test + void testAllChangeSetsRecorded() { + var mojo = new VerifyCompatibilityMojo(); + var changeSets = Set.of( + new ChangeSet("1", "keycloak", "example.xml"), + new ChangeSet("2", "keycloak", "example.xml") + ); + + assertDoesNotThrow(() -> mojo.checkMissingChangeSet(changeSets, new HashSet<>(changeSets), new File(""), new File(""))); + } + + @Test + void testMissingChangeSet() { + var mojo = new VerifyCompatibilityMojo(); + var currentChanges = new HashSet(); + currentChanges.add(new ChangeSet("1", "keycloak", "example.xml")); + currentChanges.add(new ChangeSet("2", "keycloak", "example.xml")); + + var recordedChanges = Set.of(currentChanges.iterator().next()); + + Exception e = assertThrows( + MojoExecutionException.class, + () -> mojo.checkMissingChangeSet(currentChanges, recordedChanges, new File(""), new File("")) + ); + assertEquals("One or more ChangeSet definitions are missing from the supported or unsupported files", e.getMessage()); + } +} diff --git a/misc/db-compatibility-verifier/src/test/resources/META-INF/empty-array.json b/misc/db-compatibility-verifier/src/test/resources/META-INF/empty-array.json new file mode 100644 index 00000000000..0637a088a01 --- /dev/null +++ b/misc/db-compatibility-verifier/src/test/resources/META-INF/empty-array.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/misc/db-compatibility-verifier/src/test/resources/META-INF/jpa-changelog-1.xml b/misc/db-compatibility-verifier/src/test/resources/META-INF/jpa-changelog-1.xml new file mode 100644 index 00000000000..4f7e8ca051a --- /dev/null +++ b/misc/db-compatibility-verifier/src/test/resources/META-INF/jpa-changelog-1.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/misc/db-compatibility-verifier/src/test/resources/META-INF/jpa-changelog-2.xml b/misc/db-compatibility-verifier/src/test/resources/META-INF/jpa-changelog-2.xml new file mode 100644 index 00000000000..4f7e8ca051a --- /dev/null +++ b/misc/db-compatibility-verifier/src/test/resources/META-INF/jpa-changelog-2.xml @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/misc/theme-verifier/pom.xml b/misc/theme-verifier/pom.xml index 44474101233..f8fdaf6a249 100644 --- a/misc/theme-verifier/pom.xml +++ b/misc/theme-verifier/pom.xml @@ -27,9 +27,7 @@ ../../pom.xml - org.keycloak theme-verifier-maven-plugin - 999.0.0-SNAPSHOT Keycloak Theme verifier Keycloak Theme verifier @@ -40,27 +38,25 @@ 17 17 UTF-8 - 3.15.1 - 3.9.9 org.apache.maven maven-plugin-api - ${maven-plugin-api.version} + ${maven.version} provided org.apache.maven.plugin-tools maven-plugin-annotations - ${maven-plugin-tools.version} + ${maven.plugin-tools.version} provided org.apache.maven maven-core - ${maven-plugin-api.version} + ${maven.version} compile diff --git a/model/jpa/README.md b/model/jpa/README.md new file mode 100644 index 00000000000..b1e632f5cc5 --- /dev/null +++ b/model/jpa/README.md @@ -0,0 +1,71 @@ +# Rolling updates database compatibility + +In order to track database schema changes that are compatible/incompatible with the `rolling-updates` feature, this module +makes use of the `db-compatibility-verifier-maven-plugin`. See `misc/db-compaotibility-verifier/README.md` for detailed +usage instructions. + +The rolling-update:v2 feature only supports rolling updates of Keycloak patch releases, therefore database changes +are only tracked in release branches and not `main`. + + +## Tracking supported database changes + +All Liquibase ChangeSets at branch time are considered supported by the `rolling-updates:v2` feature, as this is the +initial database state from the perspective of the current release stream. When creating a new release branch, a "snapshot" +of all known Liquibase ChangeSets in this module are recorded using the `db-compatibility-verifier:snapshot` +maven plugin. This generates two JSON files: a "supported" file with all known ChangeSets and an "unsupported" +file initialized with an empty array. Both of these files must be committed to the repository. + +A snapshot can be created by executing: + +``` +./mvnw clean install -am -pl model/jpa -Pdb-changeset-snapshot -DskipTests +``` + + +## Verifying all database changes are tracked + +The `db-compatibility-verifier:verify` plugin is used as part of the `model/jpa` test phase to ensure that +any Liquibase changeset added during the release branches lifecycle are tracked in either the supported or unsupported files. +If one of more unrecorded ChangeSet is detected, contributors need to determine if the ChangeSet is compatible with a +rolling update. If the change is not compatible, then it must be recorded in the unsupported file. Conversely, if it is +compatible it must be recorded in the supported file. + +Execution of the `db-compatibility-verifier:verify` plugin can be skipped during the test phase by specifying: `-Ddb.verify.skip=true`. + +## Adding a supported database change + +To add an individual ChangeSet to the supported file users can execute: + +``` +./mvnw -pl model/jpa org.keycloak:db-compatibility-verifier-maven-plugin:999.0.0-SNAPSHOT:supported \ + -Ddb.verify.changeset.id= \ + -Ddb.verify.changeset.author= \ + -Ddb.verify.changeset.filename= +``` + +If multiple ChangeSets exist, and they are all compatible with rolling updates, the following can be used to add all changes: + +``` +./mvnw -pl model/jpa org.keycloak:db-compatibility-verifier-maven-plugin:999.0.0-SNAPSHOT:supported \ + -Ddb.verify.changeset.addAll=true +``` + + +## Adding an unsupported database change + +To add an individual ChangeSet to the supported file users can execute: + +``` +./mvnw -pl model/jpa org.keycloak:db-compatibility-verifier-maven-plugin:999.0.0-SNAPSHOT:unsupported \ + -Ddb.verify.changeset.id= \ + -Ddb.verify.changeset.author= \ + -Ddb.verify.changeset.filename= +``` + +If multiple ChangeSets exist, and they are all compatible with rolling updates, the following can be used to add all changes: + +``` +./mvnw -pl model/jpa org.keycloak:db-compatibility-verifier-maven-plugin:999.0.0-SNAPSHOT:unsupported \ + -Ddb.verify.changeset.addAll=true +``` \ No newline at end of file diff --git a/model/jpa/pom.xml b/model/jpa/pom.xml index 0e3aa1d30e3..0f40b6f3300 100755 --- a/model/jpa/pom.xml +++ b/model/jpa/pom.xml @@ -38,6 +38,8 @@ com.h2database h2 ${h2.version} + src/main/resources/META-INF/rolling-upgrades-supported-changes.json + src/main/resources/META-INF/rolling-upgrades-unsupported-changes.json @@ -171,6 +173,44 @@ + + org.keycloak + db-compatibility-verifier-maven-plugin + ${project.version} + + + verify + test + + verify + + + + + + + + db-changeset-snapshot + + + + org.keycloak + db-compatibility-verifier-maven-plugin + ${project.version} + + + snapshot + process-resources + + snapshot + + + + + + + + diff --git a/pom.xml b/pom.xml index 2b7e542e839..6b0f7f41cdd 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ 17 17 17 + 3.15.1 999.0.0-SNAPSHOT @@ -304,6 +305,7 @@ federation services themes + misc/db-compatibility-verifier misc/theme-verifier model util diff --git a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/compatibility/DatabaseCompatibilityMetadataProvider.java b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/compatibility/DatabaseCompatibilityMetadataProvider.java index 43dc5aa6385..c0d7c570ea4 100644 --- a/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/compatibility/DatabaseCompatibilityMetadataProvider.java +++ b/quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/configuration/compatibility/DatabaseCompatibilityMetadataProvider.java @@ -1,21 +1,34 @@ package org.keycloak.quarkus.runtime.configuration.compatibility; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import org.keycloak.compatibility.CompatibilityMetadataProvider; import org.keycloak.config.DatabaseOptions; import org.keycloak.config.Option; +import org.keycloak.jose.jws.crypto.HashUtils; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import io.smallrye.config.ConfigValue; +import org.jboss.logging.Logger; import static org.keycloak.quarkus.runtime.configuration.Configuration.getConfigValue; import static org.keycloak.quarkus.runtime.configuration.Configuration.getOptionalKcValue; public class DatabaseCompatibilityMetadataProvider implements CompatibilityMetadataProvider { + private static final Logger log = Logger.getLogger(DatabaseCompatibilityMetadataProvider.class); + public static final String ID = "database"; + public static final String UNSUPPORTED_CHANGE_SET_HASH_KEY = "unsupported-changeset-hash"; @Override public Map metadata() { @@ -30,6 +43,25 @@ public class DatabaseCompatibilityMetadataProvider implements CompatibilityMetad addOptional(DatabaseOptions.DB_URL_PORT, metadata); addOptional(DatabaseOptions.DB_URL_DATABASE, metadata); } + + ObjectMapper objectMapper = new ObjectMapper(); + try (InputStream inputStream = DatabaseCompatibilityMetadataProvider.class.getResourceAsStream("/META-INF/rolling-upgrades-unsupported-changes.json")) { + if (inputStream != null) { + // Load the ChangeSet JSON into memory and write to a JSON String in order to avoid whitespace changes impacting the hash + Set changeSets = objectMapper.readValue(inputStream, new TypeReference<>() {}); + List sortedChanges = changeSets.stream().sorted( + Comparator.comparing(ChangeSet::id) + .thenComparing(ChangeSet::author) + .thenComparing(ChangeSet::filename) + ).toList(); + + String changeSetJson = objectMapper.writeValueAsString(sortedChanges); + String hash = HashUtils.sha256UrlEncodedHash(changeSetJson, StandardCharsets.UTF_8); + metadata.put(UNSUPPORTED_CHANGE_SET_HASH_KEY, hash); + } + } catch (IOException e) { + log.error("Unable to close InputStream when creating database unsupported change hash", e); + } return metadata; } @@ -42,4 +74,7 @@ public class DatabaseCompatibilityMetadataProvider implements CompatibilityMetad public String getId() { return ID; } + + public record ChangeSet(String id, String author, String filename) { + } } diff --git a/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/UpdateCommandDistTest.java b/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/UpdateCommandDistTest.java index e7e051667bd..d67fe895f24 100644 --- a/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/UpdateCommandDistTest.java +++ b/quarkus/tests/integration/src/test/java/org/keycloak/it/cli/dist/UpdateCommandDistTest.java @@ -18,10 +18,13 @@ package org.keycloak.it.cli.dist; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.Set; import org.keycloak.common.Profile; import org.keycloak.common.Version; @@ -35,6 +38,7 @@ import org.keycloak.it.junit5.extension.RawDistOnly; import org.keycloak.it.utils.KeycloakDistribution; import org.keycloak.it.utils.RawKeycloakDistribution; import org.keycloak.jgroups.certificates.DefaultJGroupsCertificateProviderFactory; +import org.keycloak.jose.jws.crypto.HashUtils; import org.keycloak.quarkus.runtime.cli.command.UpdateCompatibility; import org.keycloak.quarkus.runtime.cli.command.UpdateCompatibilityCheck; import org.keycloak.quarkus.runtime.cli.command.UpdateCompatibilityMetadata; @@ -46,15 +50,19 @@ import org.keycloak.spi.infinispan.impl.embedded.DefaultCacheEmbeddedConfigProvi import org.keycloak.spi.infinispan.impl.remote.DefaultCacheRemoteConfigProviderFactory; import org.keycloak.util.JsonSerialization; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import io.quarkus.test.junit.main.Launch; import org.junit.jupiter.api.Test; import static org.keycloak.it.cli.dist.Util.createTempFile; +import static org.keycloak.quarkus.runtime.configuration.compatibility.DatabaseCompatibilityMetadataProvider.UNSUPPORTED_CHANGE_SET_HASH_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; + @DistributionTest @RawDistOnly(reason = "Requires creating JSON file to be available between containers") public class UpdateCommandDistTest { @@ -267,9 +275,7 @@ public class UpdateCommandDistTest { var info = JsonSerialization.mapper.readValue(jsonFile, UpdateCompatibilityCheck.METADATA_TYPE_REF); var expectedMeta = defaultMeta(distribution); - expectedMeta.put(DatabaseCompatibilityMetadataProvider.ID, Map.of( - DatabaseOptions.DB.getKey(), "postgres" - )); + expectedMeta.get(DatabaseCompatibilityMetadataProvider.ID).put(DatabaseOptions.DB.getKey(), "postgres"); info.remove(FeatureCompatibilityMetadataProvider.ID); assertEquals(expectedMeta, info); @@ -287,8 +293,8 @@ public class UpdateCommandDistTest { // Assert that expected db-url-* options are written to the metadata when --db-url is not present var info = JsonSerialization.mapper.readValue(jsonFile, UpdateCompatibilityCheck.METADATA_TYPE_REF); var expectedMeta = defaultMeta(distribution); - expectedMeta.put(DatabaseCompatibilityMetadataProvider.ID, Map.of( - DatabaseOptions.DB.getKey(), DatabaseOptions.DB.getDefaultValue().get(), + var dbMeta = expectedMeta.get(DatabaseCompatibilityMetadataProvider.ID); + dbMeta.putAll(Map.of( DatabaseOptions.DB_URL_DATABASE.getKey(), "keycloak", DatabaseOptions.DB_URL_HOST.getKey(), "localhost", DatabaseOptions.DB_URL_PORT.getKey(), "9999" @@ -306,9 +312,14 @@ public class UpdateCommandDistTest { assertEquals(0, result.exitCode()); info = JsonSerialization.mapper.readValue(jsonFile, UpdateCompatibilityCheck.METADATA_TYPE_REF); - expectedMeta.put(DatabaseCompatibilityMetadataProvider.ID, Map.of( - DatabaseOptions.DB.getKey(), DatabaseOptions.DB.getDefaultValue().get() - )); + Map expectedDbMeta = new HashMap<>(); + expectedDbMeta.put(DatabaseOptions.DB.getKey(), DatabaseOptions.DB.getDefaultValue().get()); + String expectedHash = dbMeta.get(UNSUPPORTED_CHANGE_SET_HASH_KEY); + if (expectedHash != null) { + expectedDbMeta.put(UNSUPPORTED_CHANGE_SET_HASH_KEY, expectedHash); + } + expectedMeta.put(DatabaseCompatibilityMetadataProvider.ID, expectedDbMeta); + info.remove(FeatureCompatibilityMetadataProvider.ID); assertEquals(expectedMeta, info); @@ -317,15 +328,25 @@ public class UpdateCommandDistTest { result.assertExitCode(CompatibilityResult.ExitCode.ROLLING.value()); } - private Map> defaultMeta(KeycloakDistribution distribution) { + private Map> defaultMeta(KeycloakDistribution distribution) throws IOException { Map keycloak = new HashMap<>(1); keycloak.put("version", Version.VERSION); + Map dbMeta = new HashMap<>(); + dbMeta.put(DatabaseOptions.DB.getKey(), DatabaseOptions.DB.getDefaultValue().get()); + try (InputStream inputStream = UpdateCommandDistTest.class.getResourceAsStream("/META-INF/rolling-upgrades-unsupported-changes.json")) { + if (inputStream != null) { + ObjectMapper objectMapper = new ObjectMapper(); + Set changeSets = objectMapper.readValue(inputStream, new TypeReference<>() {}); + String changeSetJson = objectMapper.writeValueAsString(changeSets); + String hash = HashUtils.sha256UrlEncodedHash(changeSetJson, StandardCharsets.UTF_8); + dbMeta.put(UNSUPPORTED_CHANGE_SET_HASH_KEY, hash); + } + } + Map> m = new HashMap<>(); m.put(KeycloakCompatibilityMetadataProvider.ID, keycloak); - m.put(DatabaseCompatibilityMetadataProvider.ID, Map.of( - DatabaseOptions.DB.getKey(), DatabaseOptions.DB.getDefaultValue().get() - )); + m.put(DatabaseCompatibilityMetadataProvider.ID, dbMeta); m.put(CacheEmbeddedConfigProviderSpi.SPI_NAME, embeddedCachingMeta(distribution)); m.put(JGroupsCertificateProviderSpi.SPI_NAME, Map.of( "enabled", "true"