chore: Support for top level ref in application schemas

config/src/main/java/com/epam/aidial/core/metaschemas/DialFileFormat.java

@@ -10,6 +10,8 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static com.epam.aidial.core.metaschemas.MetaSchemaHolder.DIAL_FILE_FORMAT;
+
 public class DialFileFormat implements Format {
 
     private static final Pattern PATTERN = Pattern.compile("^files/([a-zA-Z0-9]+)/((?:(?:[a-zA-Z0-9()_\\-.~]|%[a-zA-Z0-9]{2})+/?)+)$");
@@ -43,6 +45,6 @@ private static boolean validateStringNode(ValidationContext validationContext, J
 
     @Override
     public String getName() {
-        return "dial-file-encoded";
+        return DIAL_FILE_FORMAT;
     }
 }

config/src/main/java/com/epam/aidial/core/metaschemas/MetaSchemaHolder.java

@@ -24,14 +24,19 @@ public class MetaSchemaHolder {
     public static final String PROPERTY_KIND = "dial:propertyKind";
     public static final String PROPERTY_ORDER = "dial:propertyOrder";
     public static final String APPLICATION_TYPE_ID_FIELD = "$id";
+    public static final String DIAL_FILE_KEYWORD = "dial:file";
+    public static final String DIAL_META_KEYWORD = "dial:meta";
+    public static final String PROPERTY_KIND_SERVER = "server";
+    public static final String PROPERTY_KIND_CLIENT = "client";
+    public static final String DIAL_FILE_FORMAT = "dial-file-encoded";
 
     public static String getCustomApplicationMetaSchema() {
         try (InputStream inputStream = MetaSchemaHolder.class.getClassLoader()
                 .getResourceAsStream("custom-application-schemas/schema")) {
             assert inputStream != null;
             return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
         } catch (Exception e) {
-            throw new RuntimeException("Failed to load custom application meta schema", e);
+            throw new MetaSchemaLoadException("Failed to load custom application meta schema", e);
         }
     }
 
@@ -51,4 +56,10 @@ public static JsonMetaSchema.Builder getMetaschemaBuilder() {
                 .keyword(new NonValidationKeyword("$defs"))
                 .format(new DialFileFormat());
     }
-}
+
+    public static class MetaSchemaLoadException extends RuntimeException {
+        public MetaSchemaLoadException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+}

server/build.gradle

@@ -23,12 +23,12 @@ dependencies {
     implementation 'org.apache.jclouds:jclouds-allblobstore:2.7.3'
     implementation 'org.apache.jclouds.api:filesystem:2.7.3'
     implementation 'org.redisson:redisson:3.27.0'
-    implementation 'com.networknt:json-schema-validator:1.5.2'
+    implementation 'com.networknt:json-schema-validator:1.5.7'
     implementation group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.663'
     implementation group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.12.663'
     implementation group: 'com.google.auth', name: 'google-auth-library-oauth2-http', version: '1.23.0'
     implementation group: 'com.azure', name: 'azure-identity', version: '1.13.2'
-    implementation 'org.hibernate.validator:hibernate-validator:8.0.0.Final'
+    implementation 'org.hibernate.validator:hibernate-validator:8.0.2.Final'
     implementation 'org.glassfish:jakarta.el:4.0.2'
     implementation 'jakarta.validation:jakarta.validation-api:3.0.2' // Ensure you have Jakarta Validation API dependency
     implementation 'org.apache.httpcomponents.client5:httpclient5:5.4.3'

server/src/main/java/com/epam/aidial/core/server/util/ApplicationTypeSchemaUtils.java

@@ -68,7 +68,8 @@ static String getCustomApplicationSchemaOrThrow(Config config, Application appli
     @SuppressWarnings("unchecked")
     private static Map<String, Object> filterProperties(Map<String, Object> applicationProperties, String schema, String collectorName) {
         try {
-            JsonSchema appSchema = SCHEMA_FACTORY.getSchema(schema);
+            String rewrittenSchema = JsonSchemaUtils.extractTopLevelRefs(schema);
+            JsonSchema appSchema = SCHEMA_FACTORY.getSchema(rewrittenSchema);
             CollectorContext collectorContext = new CollectorContext();
             String applicationPropertiesJson = ProxyUtil.MAPPER.writeValueAsString(applicationProperties);
             Set<ValidationMessage> validationResult = appSchema.validate(applicationPropertiesJson, InputFormat.JSON,
@@ -87,7 +88,7 @@ private static Map<String, Object> filterProperties(Map<String, Object> applicat
             return result;
         } catch (ApplicationTypeSchemaValidationException e) {
             throw e;
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new ApplicationTypeSchemaProcessingException("Failed to filter custom properties", e);
         }
     }
@@ -239,15 +240,7 @@ private static List<ResourceDescriptor> getFiles(Config config, Application appl
             }
             List<ResourceDescriptor> result = new ArrayList<>();
             for (String item : propsCollector.collect()) {
-                try {
-                    ResourceDescriptor descriptor = ResourceDescriptorFactory.fromAnyUrl(item, encryptionService);
-                    if (!descriptor.isFolder() && !resourceService.hasResource(descriptor)) {
-                        throw new ApplicationTypeResourceException("Resource listed as dependent to the application not found or inaccessible", item);
-                    }
-                    result.add(descriptor);
-                } catch (IllegalArgumentException e) {
-                    throw new ApplicationTypeResourceException("Failed to get resource descriptor for url", item, e);
-                }
+                addResourceDescriptor(result, item, encryptionService, resourceService);
             }
             return result;
         } catch (ApplicationTypeSchemaValidationException | ApplicationTypeResourceException e) {
@@ -257,6 +250,18 @@ private static List<ResourceDescriptor> getFiles(Config config, Application appl
         }
     }
 
+    private static void addResourceDescriptor(List<ResourceDescriptor> result, String item, EncryptionService encryptionService, ResourceService resourceService) {
+        try {
+            ResourceDescriptor descriptor = ResourceDescriptorFactory.fromAnyUrl(item, encryptionService);
+            if (!descriptor.isFolder() && !resourceService.hasResource(descriptor)) {
+                throw new ApplicationTypeResourceException("Resource listed as dependent to the application not found or inaccessible", item);
+            }
+            result.add(descriptor);
+        } catch (IllegalArgumentException e) {
+            throw new ApplicationTypeResourceException("Failed to get resource descriptor for url", item, e);
+        }
+    }
+
     public static Application modifySchemaRichApplication(Application application, boolean propertyFilteringRequired, ProxyContext context) {
         try {
             if (propertyFilteringRequired) {
@@ -270,4 +275,4 @@ public static Application modifySchemaRichApplication(Application application, b
         }
         return application;
     }
-}
+}

server/src/main/java/com/epam/aidial/core/server/util/JsonSchemaUtils.java

@@ -0,0 +1,107 @@
+package com.epam.aidial.core.server.util;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.experimental.UtilityClass;
+
+import java.util.Iterator;
+
+@UtilityClass
+public class JsonSchemaUtils {
+
+    private static final String PROPERTIES = "properties";
+
+    /**
+     * Resolves all first-level $ref references in the "properties" section of a JSON schema.
+     *
+     * <p>For each property with a $ref, this method:
+     * <ul>
+     *   <li>Finds the referenced node using the JSON Pointer in the $ref.</li>
+     *   <li>Merges the referenced node into the property, preserving any additional fields on the property itself (except $ref).</li>
+     *   <li>Replaces the referenced node in the schema with an empty object.</li>
+     * </ul>
+     * If no first-level property contains a $ref, the schema is returned unchanged.
+     *
+     * @param schema the JSON schema as a string
+     * @return the rewritten schema as a string with first-level $ref in properties resolved
+     * @throws JsonProcessingException if the schema cannot be parsed
+     */
+    public static String extractTopLevelRefs(String schema) throws JsonProcessingException {
+        JsonNode schemaNode = ProxyUtil.MAPPER.readTree(schema);
+        if (!hasTopLevelRefs(schemaNode)) {
+            return schema;
+        }
+        ObjectNode propertiesNode = (ObjectNode) schemaNode.get(PROPERTIES);
+        ObjectNode newPropertiesNode = propertiesNode.deepCopy();
+        boolean modified = false;
+
+        for (Iterator<String> it = propertiesNode.fieldNames(); it.hasNext(); ) {
+            String fieldName = it.next();
+            JsonNode propNode = propertiesNode.get(fieldName);
+            if (propNode.has("$ref")) {
+                String ref = propNode.get("$ref").asText();
+                // Only handle local refs; skip foreign refs
+                if (!ref.startsWith("#/")) {
+                    continue;
+                }
+                JsonNode targetNode = resolveRef(schemaNode, ref);
+                if (targetNode != null && targetNode.isObject()) {
+                    ObjectNode merged = mergeNodes(targetNode, propNode);
+                    newPropertiesNode.set(fieldName, merged);
+                    clearReferencedNode(schemaNode, ref);
+                    modified = true;
+                }
+            }
+        }
+
+        if (modified) {
+            ((ObjectNode) schemaNode).set(PROPERTIES, newPropertiesNode);
+            return ProxyUtil.MAPPER.writeValueAsString(schemaNode);
+        }
+        return schema;
+    }
+
+    private static boolean hasTopLevelRefs(JsonNode schemaNode) {
+        if (!schemaNode.has(PROPERTIES) || !schemaNode.get(PROPERTIES).isObject()) {
+            return false;
+        }
+        ObjectNode propertiesNode = (ObjectNode) schemaNode.get(PROPERTIES);
+        for (Iterator<String> it = propertiesNode.fieldNames(); it.hasNext(); ) {
+            String fieldName = it.next();
+            JsonNode propNode = propertiesNode.get(fieldName);
+            if (propNode.has("$ref")) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private static JsonNode resolveRef(JsonNode schemaNode, String ref) {
+        return schemaNode.at(ref.substring(1));
+    }
+
+    private static ObjectNode mergeNodes(JsonNode targetNode, JsonNode propNode) {
+        ObjectNode merged = ProxyUtil.MAPPER.createObjectNode();
+        targetNode.fields().forEachRemaining(e -> merged.set(e.getKey(), e.getValue()));
+        propNode.fields().forEachRemaining(e -> {
+            if (!"$ref".equals(e.getKey())) {
+                merged.set(e.getKey(), e.getValue());
+            }
+        });
+        return merged;
+    }
+
+    private static void clearReferencedNode(JsonNode schemaNode, String ref) {
+        String[] pathParts = ref.substring(1).split("/");
+        if (pathParts.length > 1) {
+            JsonNode parent = schemaNode;
+            for (int i = 1; i < pathParts.length - 1; i++) {
+                parent = parent.path(pathParts[i]);
+            }
+            if (parent.isObject()) {
+                ((ObjectNode) parent).set(pathParts[pathParts.length - 1], ProxyUtil.MAPPER.createObjectNode());
+            }
+        }
+    }
+}

server/src/main/java/com/epam/aidial/core/server/validation/DialFileKeyword.java

@@ -16,11 +16,15 @@
 import java.util.List;
 import java.util.Set;
 
+import static com.epam.aidial.core.metaschemas.MetaSchemaHolder.DIAL_FILE_KEYWORD;
+import static com.epam.aidial.core.metaschemas.MetaSchemaHolder.DIAL_META_KEYWORD;
+import static com.epam.aidial.core.metaschemas.MetaSchemaHolder.PROPERTY_KIND;
+import static com.epam.aidial.core.metaschemas.MetaSchemaHolder.PROPERTY_KIND_SERVER;
 
 public class DialFileKeyword implements Keyword {
     @Override
     public String getValue() {
-        return "dial:file";
+        return DIAL_FILE_KEYWORD;
     }
 
     @Override
@@ -30,13 +34,13 @@ public JsonValidator newValidator(SchemaLocation schemaLocation, JsonNodePath ev
     }
 
     private static class DialFileCollectorValidator extends BaseJsonValidator {
-        private static final ErrorMessageType ERROR_MESSAGE_TYPE = () -> "dial:file";
+        private static final ErrorMessageType ERROR_MESSAGE_TYPE = () -> DIAL_FILE_KEYWORD;
 
         private final Boolean value;
         private final Boolean isServerProp;
 
         private static JsonNode findMetaNode(JsonSchema schema) {
-            JsonNode metaNode = schema.getSchemaNode().get("dial:meta");
+            JsonNode metaNode = schema.getSchemaNode().get(DIAL_META_KEYWORD);
             if (metaNode != null) {
                 return metaNode;
             }
@@ -53,14 +57,14 @@ public DialFileCollectorValidator(SchemaLocation schemaLocation, JsonNodePath ev
             super(schemaLocation, evaluationPath, schemaNode, parentSchema, ERROR_MESSAGE_TYPE, keyword, validationContext, suppressSubSchemaRetrieval);
             this.value = schemaNode.booleanValue();
             JsonNode metaNode = findMetaNode(parentSchema);
-            JsonNode propertyKindNode = (metaNode != null) ? metaNode.get("dial:propertyKind") : null;
-            this.isServerProp = (propertyKindNode != null) && propertyKindNode.asText().equalsIgnoreCase("server");
+            JsonNode propertyKindNode = (metaNode != null) ? metaNode.get(PROPERTY_KIND) : null;
+            this.isServerProp = (propertyKindNode != null) && propertyKindNode.asText().equalsIgnoreCase(PROPERTY_KIND_SERVER);
         }
 
         @Override
         @SuppressWarnings("unchecked")
         public Set<ValidationMessage> validate(ExecutionContext executionContext, JsonNode jsonNode, JsonNode jsonNode1, JsonNodePath jsonNodePath) {
-            if (value) {
+            if (Boolean.TRUE.equals(value)) {
                 CollectorContext collectorContext = executionContext.getCollectorContext();
                 ListCollector<String> fileCollector = (ListCollector<String>) collectorContext.getCollectorMap()
                         .computeIfAbsent(ListCollector.FileCollectorType.ALL_FILES.getValue(), k -> new ListCollector<String>());
@@ -69,7 +73,7 @@ public Set<ValidationMessage> validate(ExecutionContext executionContext, JsonNo
                     return Set.of();
                 }
                 fileCollector.combine(List.of(nodeValue));
-                if (isServerProp) {
+                if (Boolean.TRUE.equals(isServerProp)) {
                     ListCollector<String> serverFileCollector = (ListCollector<String>) collectorContext.getCollectorMap()
                             .computeIfAbsent(ListCollector.FileCollectorType.ONLY_SERVER_FILES.getValue(), k -> new ListCollector<String>());
                     serverFileCollector.combine(List.of(nodeValue));

server/src/main/java/com/epam/aidial/core/server/validation/DialMetaKeyword.java

@@ -17,10 +17,15 @@
 import java.util.Objects;
 import java.util.Set;
 
+import static com.epam.aidial.core.metaschemas.MetaSchemaHolder.DIAL_META_KEYWORD;
+import static com.epam.aidial.core.metaschemas.MetaSchemaHolder.PROPERTY_KIND;
+import static com.epam.aidial.core.metaschemas.MetaSchemaHolder.PROPERTY_KIND_CLIENT;
+import static com.epam.aidial.core.metaschemas.MetaSchemaHolder.PROPERTY_KIND_SERVER;
+
 public class DialMetaKeyword implements Keyword {
     @Override
     public String getValue() {
-        return "dial:meta";
+        return DIAL_META_KEYWORD;
     }
 
     @Override
@@ -30,15 +35,15 @@ public JsonValidator newValidator(SchemaLocation schemaLocation, JsonNodePath ev
     }
 
     private static class DialMetaCollectorValidator extends BaseJsonValidator {
-        private static final ErrorMessageType ERROR_MESSAGE_TYPE = () -> "dial:meta";
+        private static final ErrorMessageType ERROR_MESSAGE_TYPE = () -> DIAL_META_KEYWORD;
 
         String propertyKindString;
 
         public DialMetaCollectorValidator(SchemaLocation schemaLocation, JsonNodePath evaluationPath, JsonNode schemaNode,
                                           JsonSchema parentSchema, Keyword keyword,
                                           ValidationContext validationContext, boolean suppressSubSchemaRetrieval) {
             super(schemaLocation, evaluationPath, schemaNode, parentSchema, ERROR_MESSAGE_TYPE, keyword, validationContext, suppressSubSchemaRetrieval);
-            propertyKindString = schemaNode.get("dial:propertyKind").asText();
+            propertyKindString = schemaNode.get(PROPERTY_KIND).asText();
         }
 
         @Override
@@ -47,11 +52,11 @@ public Set<ValidationMessage> validate(ExecutionContext executionContext, JsonNo
 
             CollectorContext collectorContext = executionContext.getCollectorContext();
             ListCollector<String> serverPropsCollector = (ListCollector<String>) collectorContext.getCollectorMap()
-                    .computeIfAbsent("server", k -> new ListCollector<String>());
+                    .computeIfAbsent(PROPERTY_KIND_SERVER, k -> new ListCollector<String>());
             ListCollector<String> clientPropsCollector = (ListCollector<String>) collectorContext
-                    .getCollectorMap().computeIfAbsent("client", k -> new ListCollector<String>());
+                    .getCollectorMap().computeIfAbsent(PROPERTY_KIND_CLIENT, k -> new ListCollector<String>());
             String propertyName = jsonNodePath.getName(-1);
-            if (Objects.equals(propertyKindString, "server")) {
+            if (Objects.equals(propertyKindString, PROPERTY_KIND_SERVER)) {
                 serverPropsCollector.combine(List.of(propertyName));
             } else {
                 clientPropsCollector.combine(List.of(propertyName));