Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Openshift provisioning continuing Issue #49. #52

Merged
merged 5 commits into from
Mar 19, 2024
Merged

Openshift provisioning continuing Issue #49. #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c6f6963
Add support for HA when provisioning to OpenShift
jfdenise Mar 8, 2024
30ee1ca
Initial script support, inject route as env
jfdenise Mar 13, 2024
0758f81
Smart build, initial commit
jfdenise Mar 13, 2024
84d8a5f
cleanup 2 phases build
jfdenise Mar 13, 2024
2004f81
Add labels to server imageStreams based on layer and fps
jfdenise Mar 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cli/src/main/java/org/wildfly/glow/cli/commands/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ public interface Constants {

String HELP_OPTION = "--help";
String HELP_OPTION_SHORT = "-h";
String INIT_SCRIPT_OPTION = "--init-script";
String INIT_SCRIPT_OPTION_SHORT = "-id";
String INIT_SCRIPT_OPTION_LABEL = "<init script file path>";
String INPUT_FEATURE_PACKS_FILE_OPTION = "--input-feature-packs-file";
String INPUT_FEATURE_PACKS_FILE_OPTION_LABEL = "<provisioning file path>";
String NO_DOCKER_IMAGE_OPTION = "--no-docker-image";
Expand Down
233 changes: 208 additions & 25 deletions cli/src/main/java/org/wildfly/glow/cli/commands/OpenShiftSupport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import io.fabric8.kubernetes.api.model.HTTPGetAction;
import io.fabric8.kubernetes.api.model.IntOrString;
import io.fabric8.kubernetes.api.model.ObjectReference;
import io.fabric8.kubernetes.api.model.ObjectReferenceBuilder;
import io.fabric8.kubernetes.api.model.Probe;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.ServiceBuilder;
Expand All @@ -39,16 +40,22 @@
import io.fabric8.openshift.api.model.BuildConfig;
import io.fabric8.openshift.api.model.BuildConfigBuilder;
import io.fabric8.openshift.api.model.ImageLookupPolicy;
import io.fabric8.openshift.api.model.ImageSource;
import io.fabric8.openshift.api.model.ImageSourceBuilder;
import io.fabric8.openshift.api.model.ImageSourcePath;
import io.fabric8.openshift.api.model.ImageSourcePathBuilder;
import io.fabric8.openshift.api.model.ImageStream;
import io.fabric8.openshift.api.model.ImageStreamBuilder;
import io.fabric8.openshift.api.model.Route;
import io.fabric8.openshift.api.model.RouteBuilder;
import io.fabric8.openshift.api.model.RouteTargetReference;
import io.fabric8.openshift.api.model.TLSConfig;
import io.fabric8.openshift.api.model.TagReferenceBuilder;
import io.fabric8.openshift.client.OpenShiftClient;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
Expand All @@ -59,6 +66,13 @@
import java.util.TreeMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.jboss.galleon.api.GalleonBuilder;
import org.jboss.galleon.api.Provisioning;
import org.jboss.galleon.api.config.GalleonConfigurationWithLayers;
import org.jboss.galleon.api.config.GalleonFeaturePackConfig;
import org.jboss.galleon.api.config.GalleonProvisioningConfig;
import org.jboss.galleon.config.ConfigId;
import org.jboss.galleon.util.IoUtils;
import org.jboss.galleon.util.ZipUtils;
import org.wildfly.glow.AddOn;
import org.wildfly.glow.GlowMessageWriter;
Expand Down Expand Up @@ -92,6 +106,25 @@ private static void createAppDeployment(GlowMessageWriter writer, Path target, O
for (Entry<String, String> entry : env.entrySet()) {
vars.add(new EnvVar().toBuilder().withName(entry.getKey()).withValue(entry.getValue()).build());
}
if (ha) {
writer.info("\n HA enabled, 2 replicas will be started.");
vars.add(new EnvVar().toBuilder().withName("JGROUPS_PING_PROTOCOL").withValue("DNS_PING").build());
vars.add(new EnvVar().toBuilder().withName("OPENSHIFT_DNS_PING_SERVICE_PORT").withValue("8888").build());
vars.add(new EnvVar().toBuilder().withName("OPENSHIFT_DNS_PING_SERVICE_NAME").withValue(name + "-ping").build());
IntOrString v = new IntOrString();
v.setValue(8888);
Service pingService = new ServiceBuilder().withNewMetadata().withName(name + "-ping").endMetadata().
withNewSpec().withPorts(new ServicePort().toBuilder().withProtocol("TCP").
withPort(8888).
withName("ping").
withTargetPort(v).build()).
withClusterIP("None").withPublishNotReadyAddresses().withIpFamilies("IPv4").
withInternalTrafficPolicy("Cluster").withClusterIPs("None").
withType("ClusterIP").withIpFamilyPolicy("SingleStack").
withSessionAffinity("None").withSelector(labels).endSpec().build();
osClient.services().resource(pingService).createOr(NonDeletingOperation::update);
Files.write(target.resolve(name + "-ping-service.yaml"), Serialization.asYaml(pingService).getBytes());
}
Container container = new Container();
container.setName(name);
container.setImage(name + ":latest");
Expand Down Expand Up @@ -148,7 +181,7 @@ private static void createAppDeployment(GlowMessageWriter writer, Path target, O
}

static void deploy(GlowMessageWriter writer, Path target, String appName, Map<String, String> env, Set<Layer> layers, Set<AddOn> addOns, boolean ha,
Map<String, String> extraEnv, Set<String> disabledDeployers) throws Exception {
Map<String, String> extraEnv, Set<String> disabledDeployers, Path initScript) throws Exception {
Map<String, String> actualEnv = new TreeMap<>();
OpenShiftClient osClient = new KubernetesClientBuilder().build().adapt(OpenShiftClient.class);
writer.info("\nConnected to OpenShift cluster");
Expand Down Expand Up @@ -205,8 +238,8 @@ static void deploy(GlowMessageWriter writer, Path target, String appName, Map<St
}
}

createBuild(writer, target, osClient, appName);

createBuild(writer, target, osClient, appName, initScript);
actualEnv.put("APPLICATION_ROUTE_HOST", host);
actualEnv.putAll(extraEnv);
if (!actualEnv.isEmpty()) {
if (!disabledDeployers.isEmpty()) {
Expand All @@ -222,43 +255,193 @@ static void deploy(GlowMessageWriter writer, Path target, String appName, Map<St
writer.info("\nApplication route: https://" + host + ("ROOT.war".equals(appName) ? "" : "/" + appName));
}

static void createBuild(GlowMessageWriter writer, Path target, OpenShiftClient osClient, String name, Path initScript) throws Exception {
String serverImageName = doServerImageBuild(writer, target, osClient);
doAppImageBuild(serverImageName, writer, target, osClient, name, initScript);
}

private static void packageInitScript(Path initScript, Path target) throws Exception {
Path extensions = target.resolve("extensions");
Files.createDirectories(extensions);
Path postconfigure = extensions.resolve("postconfigure.sh");
Files.copy(initScript, postconfigure);
}

private static boolean isDisabled(String name, Set<String> disabledDeployers) {
return disabledDeployers.contains("ALL") || disabledDeployers.contains(name);
}

static void createBuild(GlowMessageWriter writer, Path target, OpenShiftClient osClient, String name) throws Exception {
// zip deployment and provisioning.xml to be pushed to OpenShift
Path file = Paths.get("openshiftApp.zip");
if (Files.exists(file)) {
Files.delete(file);
private static String bytesToHex(byte[] hash) {
StringBuilder hexString = new StringBuilder(2 * hash.length);
for (int i = 0; i < hash.length; i++) {
String hex = Integer.toHexString(0xff & hash[i]);
if (hex.length() == 1) {
hexString.append('0');
}
hexString.append(hex);
}
return hexString.toString();
}

static Map<String, String> createLabels(Path provisioning) throws Exception {
GalleonBuilder provider = new GalleonBuilder();
Path dir = provisioning.getParent().resolve("tmpHome");
Files.createDirectory(dir);
StringBuilder layers = new StringBuilder();
StringBuilder excludedLayers = new StringBuilder();
StringBuilder fps = new StringBuilder();
Map<String, String> labels = new HashMap<>();
try (Provisioning p = provider.newProvisioningBuilder(provisioning).setInstallationHome(dir).build()) {
GalleonProvisioningConfig config = provider.newProvisioningBuilder(provisioning).setInstallationHome(dir).build().loadProvisioningConfig(provisioning);
GalleonConfigurationWithLayers cl = config.getDefinedConfig(new ConfigId("standalone", "standalone.xml"));
for(String s : cl.getIncludedLayers()) {
labels.put("org.wildfly.glow.layer."+s,"");
}
for(String s : cl.getExcludedLayers()) {
labels.put("org.wildfly.glow.excluded.layer."+s,"");
}
for (GalleonFeaturePackConfig gfpc : config.getFeaturePackDeps()) {
if (fps.length() != 0) {
fps.append("_");
}
String producerName = gfpc.getLocation().getProducerName();
producerName = producerName.replaceAll("::zip", "");
int i = producerName.indexOf(":");
if(i > 0) {
producerName = producerName.substring(i+1);
}
producerName = producerName.replaceAll(":", "-");
labels.put("org.wildfly.glow.feature-pack."+producerName,"");
}
}
return labels;
}

private static String format(String label) {
if (label.length() > 63) {
label = label.substring(0, 56);
label += ".trunc";
}
return label;
}
static String doServerImageBuild(GlowMessageWriter writer, Path target, OpenShiftClient osClient) throws Exception {
Path provisioning = target.resolve("galleon").resolve("provisioning.xml");
byte[] content = Files.readAllBytes(provisioning);
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] encodedhash = digest.digest(content);
String key = bytesToHex(encodedhash);
String serverImageName = "wildfly-server-" + key;

ImageStream stream = new ImageStreamBuilder().withNewMetadata().withName(serverImageName).
endMetadata().withNewSpec().withLookupPolicy(new ImageLookupPolicy(Boolean.TRUE)).endSpec().build();
// check if it exists
ImageStream existingStream = osClient.imageStreams().resource(stream).get();
if (existingStream == null) {
writer.info("\nBuilding server image (this can take up to few minutes the first time)...");
// zip deployment and provisioning.xml to be pushed to OpenShift
Path file = Paths.get("openshiftServer.zip");
if (Files.exists(file)) {
Files.delete(file);
}
file.toFile().deleteOnExit();
// First do a build of the naked server
Path stepOne = target.resolve("step-one");
Files.createDirectories(stepOne);
IoUtils.copy(target.resolve("galleon"), stepOne.resolve("galleon"));
ZipUtils.zip(stepOne, file);
stream = stream.toBuilder().editOrNewMetadata().withLabels(createLabels(provisioning)).endMetadata().build();
osClient.imageStreams().resource(stream).createOr(NonDeletingOperation::update);
Files.write(target.resolve(serverImageName + "-image-stream.yaml"), Serialization.asYaml(stream).getBytes());
BuildConfigBuilder builder = new BuildConfigBuilder();
ObjectReference ref = new ObjectReference();
ref.setKind("ImageStreamTag");
ref.setName(serverImageName + ":latest");
BuildConfig buildConfig = builder.
withNewMetadata().withName(serverImageName + "-build").endMetadata().withNewSpec().
withNewOutput().
withNewTo().
withKind("ImageStreamTag").
withName(serverImageName + ":latest").endTo().
endOutput().withNewStrategy().withNewSourceStrategy().withNewFrom().withKind("DockerImage").
withName("quay.io/wildfly/wildfly-s2i:latest").endFrom().
withIncremental(true).
withEnv(new EnvVar().toBuilder().withName("GALLEON_USE_LOCAL_FILE").withValue("true").build()).
endSourceStrategy().endStrategy().withNewSource().
withType("Binary").endSource().endSpec().build();
osClient.buildConfigs().resource(buildConfig).createOr(NonDeletingOperation::update);
Files.write(target.resolve(serverImageName + "-build-config.yaml"), Serialization.asYaml(buildConfig).getBytes());

Build build = osClient.buildConfigs().withName(serverImageName + "-build").instantiateBinary().fromFile(file.toFile());
CountDownLatch latch = new CountDownLatch(1);
try (Watch watcher = osClient.builds().withName(build.getMetadata().getName()).watch(getBuildWatcher(writer, latch))) {
latch.await();
}
}
file.toFile().deleteOnExit();
ZipUtils.zip(target, file);
writer.info("\nCreating and starting application image build on OpenShift (this can take up to few minutes)...");
ImageStream stream = new ImageStreamBuilder().withNewMetadata().withName(name).
return serverImageName;
}

static void doAppImageBuild(String serverImageName, GlowMessageWriter writer, Path target, OpenShiftClient osClient, String name, Path initScript) throws Exception {
// Now step 2
// From the server image, do a docker build, copy the server and copy in it the deployments and init file.
Path stepTwo = target.resolve("step-two");
IoUtils.copy(target.resolve("deployments"), stepTwo.resolve("deployments"));
StringBuilder dockerFileBuilder = new StringBuilder();
dockerFileBuilder.append("FROM wildfly-runtime:latest\n");
dockerFileBuilder.append("COPY --chown=jboss:root /server $JBOSS_HOME\n");
dockerFileBuilder.append("COPY --chown=jboss:root deployments/* $JBOSS_HOME/standalone/deployments\n");

if (initScript != null) {
packageInitScript(initScript, stepTwo);
dockerFileBuilder.append("COPY --chown=jboss:root extensions $JBOSS_HOME/extensions\n");
dockerFileBuilder.append("RUN chmod ug+rwx $JBOSS_HOME/extensions/postconfigure.sh\n");
}

dockerFileBuilder.append("RUN chmod -R ug+rwX $JBOSS_HOME\n");

Path dockerFile = stepTwo.resolve("Dockerfile");
Files.write(dockerFile, dockerFileBuilder.toString().getBytes());
Path file2 = Paths.get("openshiftApp.zip");
if (Files.exists(file2)) {
Files.delete(file2);
}
ZipUtils.zip(stepTwo, file2);
writer.info("\nCreating and starting application image build on OpenShift...");
ImageStream runtimeStream = new ImageStreamBuilder().withNewMetadata().withName("wildfly-runtime").
endMetadata().withNewSpec().
addToTags(0, new TagReferenceBuilder()
.withName("latest")
.withFrom(new ObjectReferenceBuilder()
.withKind("DockerImage")
.withName("quay.io/wildfly/wildfly-runtime:latest")
.build())
.build()).
withLookupPolicy(new ImageLookupPolicy(Boolean.TRUE)).endSpec().build();
osClient.imageStreams().resource(runtimeStream).createOr(NonDeletingOperation::update);
ImageStream appStream = new ImageStreamBuilder().withNewMetadata().withName(name).
endMetadata().withNewSpec().withLookupPolicy(new ImageLookupPolicy(Boolean.TRUE)).endSpec().build();
osClient.imageStreams().resource(stream).createOr(NonDeletingOperation::update);
Files.write(target.resolve(name + "-image-stream.yaml"), Serialization.asYaml(stream).getBytes());
osClient.imageStreams().resource(appStream).createOr(NonDeletingOperation::update);
BuildConfigBuilder builder = new BuildConfigBuilder();
ObjectReference ref = new ObjectReference();
ref.setKind("ImageStreamTag");
ref.setName(name + ":latest");
BuildConfig buildConfig = builder.
ref.setName(serverImageName + ":latest");
ImageSourcePath srcPath = new ImageSourcePathBuilder().withSourcePath("/opt/server").withDestinationDir(".").build();
ImageSource imageSource = new ImageSourceBuilder().withFrom(ref).withPaths(srcPath).build();
BuildConfig buildConfig2 = builder.
withNewMetadata().withName(name + "-build").endMetadata().withNewSpec().
withNewOutput().
withNewTo().
withKind("ImageStreamTag").
withName(name + ":latest").endTo().
endOutput().withNewStrategy().withNewSourceStrategy().withNewFrom().withKind("DockerImage").
withName("quay.io/wildfly/wildfly-s2i:latest").endFrom().
withIncremental(true).
withEnv(new EnvVar().toBuilder().withName("GALLEON_USE_LOCAL_FILE").withValue("true").build()).
endSourceStrategy().endStrategy().withNewSource().
withType("Binary").endSource().endSpec().build();
osClient.buildConfigs().resource(buildConfig).createOr(NonDeletingOperation::update);
Files.write(target.resolve(name + "-build-config.yaml"), Serialization.asYaml(buildConfig).getBytes());
endOutput().
withNewSource().withType("Binary").withImages(imageSource).endSource().
withNewStrategy().withNewDockerStrategy().withNewFrom().withKind("ImageStream").
withName("wildfly-runtime").endFrom().
withDockerfilePath("./Dockerfile").
endDockerStrategy().endStrategy().endSpec().build();
osClient.buildConfigs().resource(buildConfig2).createOr(NonDeletingOperation::update);
Files.write(target.resolve(name + "-build-config.yaml"), Serialization.asYaml(buildConfig2).getBytes());

Build build = osClient.buildConfigs().withName(name + "-build").instantiateBinary().fromFile(file.toFile());
Build build = osClient.buildConfigs().withName(name + "-build").instantiateBinary().fromFile(file2.toFile());
CountDownLatch latch = new CountDownLatch(1);
try (Watch watcher = osClient.builds().withName(build.getMetadata().getName()).watch(getBuildWatcher(writer, latch))) {
latch.await();
Expand Down
Loading
Loading