Refactor to IoUtils; Cover FileResolver, StackFrameTransformer; more …

…tests

pkl-cli/src/test/kotlin/org/pkl/cli/CliEvaluatorTest.kt

@@ -1457,6 +1457,49 @@ result = someLib.x
  assertThat(output).isEqualTo("result = 1\n")
  }
 
+ @Test
+ fun `eval file with non-ASCII name`() {
+ val dir = tempDir.resolve("🤬").createDirectory()
+ val file = writePklFile(dir.resolve("日本語.pkl").toString(), """
+ 日本語 = "Japanese language"
+ readDir = read(".").text
+ readDirFile = read("file:$tempDir/🤬").text
+ readOne = read("日本語.pkl").text.split("\n").first
+ readOneFile = read("file:$tempDir/🤬/日本語.pkl").text.split("\n").first
+ readGlob = read*("./日*.pkl").keys
+ readGlobFile = read*("file:$tempDir/**/*.pkl").keys.map((it) -> it.replaceAll("$tempDir", ""))
+ importOne = import("日本語.pkl").readOne
+ importOneFile = import("file:$tempDir/🤬/日本語.pkl").日本語
+ importGlob = import*("./日*.pkl").keys
+ importGlobFile = import*("file:$tempDir/**/*.pkl").keys.map((it) -> it.replaceAll("$tempDir", ""))
+ """.trimIndent())
+ val output = 
+ evalToConsole(
+ CliEvaluatorOptions(
+ CliBaseOptions(sourceModules = listOf(file)),
+ )
+ )
+
+ assertThat(output).isEqualTo("""日本語 = "Japanese language"
+readDir = ""${'"'}
+ 日本語.pkl
+ 
+ ""${'"'}
+readDirFile = ""${'"'}
+ 日本語.pkl
+ 
+ ""${'"'}
+readOne = "日本語 = \"Japanese language\""
+readOneFile = "日本語 = \"Japanese language\""
+readGlob = Set("./日本語.pkl")
+readGlobFile = Set("file:/🤬/日本語.pkl")
+importOne = "日本語 = \"Japanese language\""
+importOneFile = "Japanese language"
+importGlob = Set("./日本語.pkl")
+importGlobFile = Set("file:/🤬/日本語.pkl")
+""")
+ }
+
  private fun evalModuleThatImportsPackage(certsFile: Path?, testPort: Int = -1) {
  val moduleUri =
  writePklFile(

pkl-core/src/main/java/org/pkl/core/StackFrameTransformers.java

@@ -93,7 +93,7 @@ public static StackFrameTransformer convertFilePathToUriScheme(String scheme) {
  var uri = frame.getModuleUri();
  if (!uri.startsWith("file:")) return frame;
 
- return transformUri(frame, Path.of(URI.create(uri)).toString(), scheme);
+ return transformUri(frame, IoUtils.pathOfURI(URI.create(uri)).toString(), scheme);
  };
  }
 

pkl-core/src/main/java/org/pkl/core/module/FileResolver.java

@@ -24,12 +24,13 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import org.pkl.core.util.IoUtils;
 
 public final class FileResolver {
  private FileResolver() {}
 
  public static List<PathElement> listElements(URI baseUri) throws IOException {
- return listElements(Path.of(baseUri));
+ return listElements(IoUtils.pathOfURI(baseUri));
  }
 
  public static List<PathElement> listElements(Path path) throws IOException {
@@ -49,7 +50,7 @@ public static List<PathElement> listElements(Path path) throws IOException {
  }
 
  public static boolean hasElement(URI elementUri) {
- return Files.exists(Path.of(elementUri));
+ return Files.exists(IoUtils.pathOfURI(elementUri));
  }
 
  public static boolean hasElement(Path path) {

pkl-core/src/main/java/org/pkl/core/module/ModuleKeys.java

@@ -327,18 +327,10 @@ public ResolvedModuleKey resolve(SecurityManager securityManager)
  throw new FileNotFoundException();
  }
 
- // Path.of(URI) throws on non-ASCII characters so the module URI here must be normalized to
- // ASCII
- // Unfortunately there's no way to go from URI -> ASCII URI directly
- // so this must transform URI -> ASCII String -> ASCII URI
- try {
- var realPath = Path.of(new URI(uri.toASCIIString())).toRealPath();
- var resolvedUri = realPath.toUri();
- securityManager.checkResolveModule(resolvedUri);
- return ResolvedModuleKeys.file(this, resolvedUri, realPath);
- } catch (URISyntaxException e) {
- throw new PklBugException("File module URI could not be normalized to ASCII", e);
- }
+ var realPath = IoUtils.pathOfURI(uri).toRealPath();
+ var resolvedUri = realPath.toUri();
+ securityManager.checkResolveModule(resolvedUri);
+ return ResolvedModuleKeys.file(this, resolvedUri, realPath);
  }
 
  @Override

pkl-core/src/main/java/org/pkl/core/util/IoUtils.java

@@ -39,6 +39,7 @@
 import org.pkl.core.SecurityManager;
 import org.pkl.core.SecurityManagerException;
 import org.pkl.core.module.ModuleKey;
+import org.pkl.core.module.ResolvedModuleKeys;
 import org.pkl.core.packages.PackageLoadError;
 import org.pkl.core.runtime.ReaderBase;
 import org.pkl.core.runtime.VmContext;
@@ -88,6 +89,22 @@ public static URI toUri(String str) throws URISyntaxException {
  return new URI(null, null, str, null);
  }
 
+ /**
+ * Converts a URI to a Path, normalizing any non-ASCII characters.
+ */
+ public static Path pathOfURI(URI uri) {
+ // Path.of(URI) throws on non-ASCII characters so the module URI here must be normalized to
+ // ASCII
+ // Unfortunately there's no way to go from URI -> ASCII URI directly
+ // so this must transform URI -> ASCII String -> ASCII URI
+ try {
+ return Path.of(new URI(uri.toASCIIString()));
+ } catch (URISyntaxException e) {
+ // impossible to get here; we started from a valid URI to begin with
+ throw PklBugException.unreachableCode();
+ }
+ }
+
  /** Like {@link #toUri(String)}, except without checked exceptions. */
  public static URI createUri(String str) {
  try {

pkl-core/src/test/files/LanguageSnippetTests/input/modules/日本語_error.pkl

@@ -0,0 +1,2 @@
+// covers https://github.com/apple/pkl/issues/653
+日本語 = throw("Error reporting should also handle unicode filenames!")

pkl-core/src/test/files/LanguageSnippetTests/output/basic/readGlob.pcf

@@ -109,11 +109,15 @@ examples {
  }
  ["prop:"] {
  new {
+ ["prop:inputFile"] = "/$snippetsDir/input/basic/readGlob.pkl"
+ ["prop:inputFileDirectory"] = "/$snippetsDir/input/basic/readGlob.pkl"
  ["prop:name1"] = "value1"
  ["prop:name2"] = "value2"
  }
  new {
  ["prop:/foo/bar"] = "foobar"
+ ["prop:inputFile"] = "/$snippetsDir/input/basic/readGlob.pkl"
+ ["prop:inputFileDirectory"] = "/$snippetsDir/input/basic/readGlob.pkl"
  ["prop:name1"] = "value1"
  ["prop:name2"] = "value2"
  }

pkl-core/src/test/files/LanguageSnippetTests/output/modules/日本語.pcf

@@ -8,7 +8,7 @@ readOne = """
  importGlob = import*("./日*.pkl").keys
  
  """
-readGlob = Set("./日本語.pkl")
+readGlob = Set("./日本語.pkl", "./日本語_error.pkl")
 importOne = """
  // covers https://github.com/apple/pkl/issues/653
  日本語 = "Japanese language"
@@ -18,4 +18,4 @@ importOne = """
  importGlob = import*("./日*.pkl").keys
  
  """
-importGlob = Set("./日本語.pkl")
+importGlob = Set("./日本語.pkl", "./日本語_error.pkl")

pkl-core/src/test/files/LanguageSnippetTests/output/modules/日本語_error.err

@@ -0,0 +1,10 @@
+–– Pkl Error ––
+Error reporting should also handle unicode filenames!
+
+x | 日本語 = throw("Error reporting should also handle unicode filenames!")
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+at 日本語_error#日本語 (file:///$snippetsDir/input/modules/%E6%97%A5%E6%9C%AC%E8%AA%9E_error.pkl)
+
+xxx | text = renderer.renderDocument(value)
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+at pkl.base#Module.output.text (pkl:base)

pkl-core/src/test/files/LanguageSnippetTests/output/projects/evaluatorSettings/basic.pcf

@@ -6,6 +6,8 @@ examples {
  }
  ["all props come from settings"] {
  new {
+ ["prop:inputFile"] = "/$snippetsDir/input/projects/evaluatorSettings/basic.pkl"
+ ["prop:inputFileDirectory"] = "/$snippetsDir/input/projects/evaluatorSettings/basic.pkl"
  ["prop:two"] = "2"
  }
  }