Skip to content

Commit e776480

Browse files
committed
Preserve undecodable launcher argv bytes
1 parent e53a261 commit e776480

3 files changed

Lines changed: 72 additions & 5 deletions

File tree

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@
3434
import java.io.InputStream;
3535
import java.io.OutputStream;
3636
import java.lang.management.ManagementFactory;
37+
import java.nio.ByteBuffer;
38+
import java.nio.CharBuffer;
39+
import java.nio.charset.Charset;
40+
import java.nio.charset.CharsetDecoder;
41+
import java.nio.charset.CoderResult;
42+
import java.nio.charset.CodingErrorAction;
3743
import java.nio.file.Files;
3844
import java.nio.file.InvalidPathException;
3945
import java.nio.file.NoSuchFileException;
@@ -80,6 +86,56 @@ public static void main(String[] args) {
8086
new GraalPythonMain().launch(args);
8187
}
8288

89+
@Override
90+
protected String decodeArgument(byte[] argument) {
91+
Charset charset = Charset.defaultCharset();
92+
String decoded = new String(argument, charset);
93+
if (decoded.indexOf('\uFFFD') < 0) {
94+
return decoded;
95+
}
96+
return decodeArgument(argument, charset);
97+
}
98+
99+
/*
100+
* Match documented Unix sys.argv behavior:
101+
* https://docs.python.org/3.12/library/sys.html#sys.argv
102+
* "When you need original bytes, you can get it by `[os.fsencode(arg) for arg in sys.argv]`."
103+
*/
104+
private static String decodeArgument(byte[] argument, Charset charset) {
105+
CharsetDecoder decoder = charset.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT);
106+
StringBuilder builder = new StringBuilder(argument.length);
107+
ByteBuffer in = ByteBuffer.wrap(argument);
108+
CharBuffer out = CharBuffer.allocate(Math.max(1, (int) Math.ceil(argument.length * decoder.maxCharsPerByte())));
109+
while (true) {
110+
CoderResult result = decoder.decode(in, out, true);
111+
appendDecodedChars(builder, out);
112+
if (result.isUnderflow()) {
113+
break;
114+
} else if (result.isOverflow()) {
115+
continue;
116+
} else if (result.isError()) {
117+
int errorLength = result.length();
118+
for (int i = 0; i < errorLength && in.hasRemaining(); i++) {
119+
builder.append((char) (0xDC00 + (in.get() & 0xff)));
120+
}
121+
}
122+
}
123+
while (true) {
124+
CoderResult result = decoder.flush(out);
125+
appendDecodedChars(builder, out);
126+
if (result.isUnderflow()) {
127+
break;
128+
}
129+
}
130+
return builder.toString();
131+
}
132+
133+
private static void appendDecodedChars(StringBuilder builder, CharBuffer out) {
134+
out.flip();
135+
builder.append(out);
136+
out.clear();
137+
}
138+
83139
private static final String LANGUAGE_ID = "python";
84140

85141
private static final String J_PYENVCFG = "pyvenv.cfg";

‎graalpython/com.oracle.graal.python.test/src/tests/test_subprocess.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, 2025, Oracle and/or its affiliates.
1+
# Copyright (c) 2018, 2026, Oracle and/or its affiliates.
22
# Copyright (C) 1996-2017 Python Software Foundation
33
#
44
# Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -75,6 +75,17 @@ def test_check_output(self):
7575
[sys.executable, "-c", "print('BDFL')"])
7676
self.assertIn(b'BDFL', output)
7777

78+
@unittest.skipIf(sys.platform == 'win32', "POSIX argv bytes specific")
79+
def test_surrogateescape_non_utf8_argv(self):
80+
code = (
81+
"import os, sys; "
82+
"assert os.fsencode(sys.argv[-1]) == b'\\x8av'; "
83+
"print(repr(sys.argv[-1]))"
84+
)
85+
cmd = f"{shlex.quote(sys.executable)} -c {shlex.quote(code)} \"$(printf '\\212v')\""
86+
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.PIPE, text=True)
87+
self.assertEqual("'\\udc8av'\n", output)
88+
7889
def test_check_output_nonzero(self):
7990
# check_call() function with non-zero return code
8091
with self.assertRaises(subprocess.CalledProcessError) as c:

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ast/Validator.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -1006,9 +1006,9 @@ public Void visit(TypeVarTuple node) {
10061006
return null;
10071007
}
10081008

1009-
/*-
1010-
// Validation of sequences
1011-
*/
1009+
/*-
1010+
// Validation of sequences
1011+
*/
10121012

10131013
// Equivalent of validate_stmts
10141014
private void validateStmts(StmtTy[] stmts) {

0 commit comments

Comments
 (0)