diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index d4d1a8c2..74c5d2f2 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -1354,49 +1354,7 @@ private void executeTuples(PositionTracker position) case DIV_EQ_ARRAY: case MOD_EQ_ARRAY: case POW_EQ_ARRAY: { - // arg[0] = offset - // arg[1] = isGlobal - // stack[0] = array index - // stack[1] = value - Object arrIdx = pop(); - Object rhs = pop(); - if (rhs == null) { - rhs = BLANK; - } - VariableTuple variableTuple = (VariableTuple) tuple; - long offset = variableTuple.getVariableOffset(); - boolean isGlobal = variableTuple.isGlobal(); - - Map array = ensureMapVariable(offset, isGlobal); - checkScalar(arrIdx); - Object o = blankToZero(array.get(arrIdx)); - - Object newVal; - - switch (opcode) { - case PLUS_EQ_ARRAY: - newVal = JRT.add(o, rhs); - break; - case MINUS_EQ_ARRAY: - newVal = JRT.subtract(o, rhs); - break; - case MULT_EQ_ARRAY: - newVal = JRT.multiply(o, rhs); - break; - case DIV_EQ_ARRAY: - newVal = JRT.divide(o, rhs); - break; - case MOD_EQ_ARRAY: - newVal = JRT.mod(o, rhs); - break; - case POW_EQ_ARRAY: - newVal = JRT.pow(o, rhs); - break; - default: - throw new Error("Invalid op code here: " + opcode); - } - - assignArray(offset, arrIdx, newVal, isGlobal); + execCompoundAssignArray(opcode, (VariableTuple) tuple); position.next(); break; } @@ -1406,44 +1364,7 @@ private void executeTuples(PositionTracker position) case DIV_EQ_MAP_ELEMENT: case MOD_EQ_MAP_ELEMENT: case POW_EQ_MAP_ELEMENT: { - // stack[0] = array index - // stack[1] = associative array - // stack[2] = value - Object arrIdx = pop(); - Map array = toMap(pop()); - Object rhs = pop(); - if (rhs == null) { - rhs = BLANK; - } - - checkScalar(arrIdx); - Object o = blankToZero(array.get(arrIdx)); - Object newVal; - - switch (opcode) { - case PLUS_EQ_MAP_ELEMENT: - newVal = JRT.add(o, rhs); - break; - case MINUS_EQ_MAP_ELEMENT: - newVal = JRT.subtract(o, rhs); - break; - case MULT_EQ_MAP_ELEMENT: - newVal = JRT.multiply(o, rhs); - break; - case DIV_EQ_MAP_ELEMENT: - newVal = JRT.divide(o, rhs); - break; - case MOD_EQ_MAP_ELEMENT: - newVal = JRT.mod(o, rhs); - break; - case POW_EQ_MAP_ELEMENT: - newVal = JRT.pow(o, rhs); - break; - default: - throw new Error("Invalid op code here: " + opcode); - } - - assignMapElement(array, arrIdx, newVal); + execCompoundAssignMapElement(opcode); position.next(); break; } @@ -1478,39 +1399,7 @@ private void executeTuples(PositionTracker position) case DIV_EQ: case MOD_EQ: case POW_EQ: { - // arg[0] = offset - // arg[1] = isGlobal - // stack[0] = value - VariableTuple variableTuple = (VariableTuple) tuple; - long offset = variableTuple.getVariableOffset(); - boolean isGlobal = variableTuple.isGlobal(); - Object o1 = blankToZero(resolveVariable(offset, isGlobal, false)); - Object o2 = pop(); - Object ans; - switch (opcode) { - case PLUS_EQ: - ans = JRT.add(o1, o2); - break; - case MINUS_EQ: - ans = JRT.subtract(o1, o2); - break; - case MULT_EQ: - ans = JRT.multiply(o1, o2); - break; - case DIV_EQ: - ans = JRT.divide(o1, o2); - break; - case MOD_EQ: - ans = JRT.mod(o1, o2); - break; - case POW_EQ: - ans = JRT.pow(o1, o2); - break; - default: - throw new Error("Invalid opcode here: " + opcode); - } - push(ans); - runtimeStack.setVariable(offset, ans, isGlobal); + execCompoundAssignVariable(opcode, (VariableTuple) tuple); position.next(); break; } @@ -1520,44 +1409,8 @@ private void executeTuples(PositionTracker position) case DIV_EQ_INPUT_FIELD: case MOD_EQ_INPUT_FIELD: case POW_EQ_INPUT_FIELD: { - // stack[0] = dollar_fieldNumber - // stack[1] = inc value - - // same code as GET_INPUT_FIELD: - long fieldnum = JRT.parseFieldNumber(pop()); - Object incval = pop(); - - // except here, get the number, and add the incvalue - Object numObj = blankToZero(jrt.jrtGetInputField(fieldnum)); - Object num; - switch (opcode) { - case PLUS_EQ_INPUT_FIELD: - num = JRT.add(numObj, incval); - break; - case MINUS_EQ_INPUT_FIELD: - num = JRT.subtract(numObj, incval); - break; - case MULT_EQ_INPUT_FIELD: - num = JRT.multiply(numObj, incval); - break; - case DIV_EQ_INPUT_FIELD: - num = JRT.divide(numObj, incval); - break; - case MOD_EQ_INPUT_FIELD: - num = JRT.mod(numObj, incval); - break; - case POW_EQ_INPUT_FIELD: - num = JRT.pow(numObj, incval); - break; - default: - throw new Error("Invalid opcode here: " + opcode); - } - setNumOnJRT(fieldnum, num); - - // put the result value on the stack - push(num); + execCompoundAssignInputField(opcode); position.next(); - break; } case INC: { @@ -2214,70 +2067,8 @@ private void executeTuples(PositionTracker position) break; } case INDIRECT_CALL: { - IndirectCallTuple callTuple = (IndirectCallTuple) tuple; - Object[] actualArguments = popArguments(callTuple.getNumActualParams()); - String requestedName = jrt.toAwkString(pop()); - String qualifiedName = normalizeIndirectFunctionName(requestedName); - IndirectFunctionTarget target = callTuple.getUserFunctions().get(qualifiedName); - if (target != null) { - long formalCount = target.getNumFormalParams(); - if (actualArguments.length > formalCount) { - jrt - .printWarning( - "gawk: " - + callTuple.getSourceName() - + ":" - + callTuple.getSourceLine() - + ": warning: function `" - + qualifiedName - + "' called with more arguments than declared"); - } - if (profiling) { - activeProfilingFunctions.push(new ActiveFunction(qualifiedName, tupleStartNanos)); - } - runtimeStack.pushFrame(formalCount, position.currentIndex()); - adoptElementArgumentReferences(actualArguments); - int copiedArgumentCount = Math.min(actualArguments.length, (int) formalCount); - for (int i = 0; i < copiedArgumentCount; i++) { - runtimeStack.setVariable(i, actualArguments[i], false); - } - position.jump(target.getAddress()); - break; - } - - String awkName = requestedName.startsWith("awk::") ? - requestedName.substring("awk::".length()) : requestedName; - BuiltinFunction builtin = BuiltinFunction.of(awkName); - if (builtin != null) { - resolveIndirectArguments(actualArguments, builtin); - push(invokeIndirectBuiltin(builtin, actualArguments, position.lineNumber())); - position.next(); - break; - } - ExtensionFunction extensionFunction = callTuple.getExtensionFunctions().get(awkName); - if (extensionFunction != null) { - resolveIndirectArguments(actualArguments, extensionFunction); - if (profiling) { - activeProfilingFunctions.push(new ActiveFunction(awkName, tupleStartNanos)); - } - try { - push( - invokeExtension( - extensionFunction, - actualArguments, - position.lineNumber(), - true)); - } finally { - if (profiling) { - recordFunctionExit(System.nanoTime()); - } - } - position.next(); - break; - } - throw new AwkRuntimeException( - position.lineNumber(), - "function `" + qualifiedName + "' is not defined"); + execIndirectCall((IndirectCallTuple) tuple, position, tupleStartNanos); + break; } case FUNCTION: { // important for compilation, @@ -2793,6 +2584,236 @@ public ProfilingReport getProfilingReport() { return new ProfilingReport(tupleProfilingStats, functionProfilingStats); } + // The exec* helpers below are extracted from executeTuples on purpose: + // that method must stay well under HotSpot's HugeMethodLimit (8000 + // bytecodes) or the JIT never compiles the interpreter loop (see #562). + + private void execCompoundAssignVariable(Opcode opcode, VariableTuple variableTuple) { + // arg[0] = offset + // arg[1] = isGlobal + // stack[0] = value + long offset = variableTuple.getVariableOffset(); + boolean isGlobal = variableTuple.isGlobal(); + Object o1 = blankToZero(resolveVariable(offset, isGlobal, false)); + Object o2 = pop(); + Object ans; + switch (opcode) { + case PLUS_EQ: + ans = JRT.add(o1, o2); + break; + case MINUS_EQ: + ans = JRT.subtract(o1, o2); + break; + case MULT_EQ: + ans = JRT.multiply(o1, o2); + break; + case DIV_EQ: + ans = JRT.divide(o1, o2); + break; + case MOD_EQ: + ans = JRT.mod(o1, o2); + break; + case POW_EQ: + ans = JRT.pow(o1, o2); + break; + default: + throw new Error("Invalid opcode here: " + opcode); + } + push(ans); + runtimeStack.setVariable(offset, ans, isGlobal); + } + + private void execCompoundAssignArray(Opcode opcode, VariableTuple variableTuple) { + // arg[0] = offset + // arg[1] = isGlobal + // stack[0] = array index + // stack[1] = value + Object arrIdx = pop(); + Object rhs = pop(); + if (rhs == null) { + rhs = BLANK; + } + long offset = variableTuple.getVariableOffset(); + boolean isGlobal = variableTuple.isGlobal(); + + Map array = ensureMapVariable(offset, isGlobal); + checkScalar(arrIdx); + Object o = blankToZero(array.get(arrIdx)); + + Object newVal; + + switch (opcode) { + case PLUS_EQ_ARRAY: + newVal = JRT.add(o, rhs); + break; + case MINUS_EQ_ARRAY: + newVal = JRT.subtract(o, rhs); + break; + case MULT_EQ_ARRAY: + newVal = JRT.multiply(o, rhs); + break; + case DIV_EQ_ARRAY: + newVal = JRT.divide(o, rhs); + break; + case MOD_EQ_ARRAY: + newVal = JRT.mod(o, rhs); + break; + case POW_EQ_ARRAY: + newVal = JRT.pow(o, rhs); + break; + default: + throw new Error("Invalid op code here: " + opcode); + } + + assignArray(offset, arrIdx, newVal, isGlobal); + } + + private void execCompoundAssignMapElement(Opcode opcode) { + // stack[0] = array index + // stack[1] = associative array + // stack[2] = value + Object arrIdx = pop(); + Map array = toMap(pop()); + Object rhs = pop(); + if (rhs == null) { + rhs = BLANK; + } + + checkScalar(arrIdx); + Object o = blankToZero(array.get(arrIdx)); + Object newVal; + + switch (opcode) { + case PLUS_EQ_MAP_ELEMENT: + newVal = JRT.add(o, rhs); + break; + case MINUS_EQ_MAP_ELEMENT: + newVal = JRT.subtract(o, rhs); + break; + case MULT_EQ_MAP_ELEMENT: + newVal = JRT.multiply(o, rhs); + break; + case DIV_EQ_MAP_ELEMENT: + newVal = JRT.divide(o, rhs); + break; + case MOD_EQ_MAP_ELEMENT: + newVal = JRT.mod(o, rhs); + break; + case POW_EQ_MAP_ELEMENT: + newVal = JRT.pow(o, rhs); + break; + default: + throw new Error("Invalid op code here: " + opcode); + } + + assignMapElement(array, arrIdx, newVal); + } + + private void execCompoundAssignInputField(Opcode opcode) { + // stack[0] = dollar_fieldNumber + // stack[1] = inc value + + // same code as GET_INPUT_FIELD: + long fieldnum = JRT.parseFieldNumber(pop()); + Object incval = pop(); + + // except here, get the number, and add the incvalue + Object numObj = blankToZero(jrt.jrtGetInputField(fieldnum)); + Object num; + switch (opcode) { + case PLUS_EQ_INPUT_FIELD: + num = JRT.add(numObj, incval); + break; + case MINUS_EQ_INPUT_FIELD: + num = JRT.subtract(numObj, incval); + break; + case MULT_EQ_INPUT_FIELD: + num = JRT.multiply(numObj, incval); + break; + case DIV_EQ_INPUT_FIELD: + num = JRT.divide(numObj, incval); + break; + case MOD_EQ_INPUT_FIELD: + num = JRT.mod(numObj, incval); + break; + case POW_EQ_INPUT_FIELD: + num = JRT.pow(numObj, incval); + break; + default: + throw new Error("Invalid opcode here: " + opcode); + } + setNumOnJRT(fieldnum, num); + + // put the result value on the stack + push(num); + } + + private void execIndirectCall(IndirectCallTuple callTuple, PositionTracker position, long tupleStartNanos) { + Object[] actualArguments = popArguments(callTuple.getNumActualParams()); + String requestedName = jrt.toAwkString(pop()); + String qualifiedName = normalizeIndirectFunctionName(requestedName); + IndirectFunctionTarget target = callTuple.getUserFunctions().get(qualifiedName); + if (target != null) { + long formalCount = target.getNumFormalParams(); + if (actualArguments.length > formalCount) { + jrt + .printWarning( + "gawk: " + + callTuple.getSourceName() + + ":" + + callTuple.getSourceLine() + + ": warning: function `" + + qualifiedName + + "' called with more arguments than declared"); + } + if (profiling) { + activeProfilingFunctions.push(new ActiveFunction(qualifiedName, tupleStartNanos)); + } + runtimeStack.pushFrame(formalCount, position.currentIndex()); + adoptElementArgumentReferences(actualArguments); + int copiedArgumentCount = Math.min(actualArguments.length, (int) formalCount); + for (int i = 0; i < copiedArgumentCount; i++) { + runtimeStack.setVariable(i, actualArguments[i], false); + } + position.jump(target.getAddress()); + return; + } + + String awkName = requestedName.startsWith("awk::") ? + requestedName.substring("awk::".length()) : requestedName; + BuiltinFunction builtin = BuiltinFunction.of(awkName); + if (builtin != null) { + resolveIndirectArguments(actualArguments, builtin); + push(invokeIndirectBuiltin(builtin, actualArguments, position.lineNumber())); + position.next(); + return; + } + ExtensionFunction extensionFunction = callTuple.getExtensionFunctions().get(awkName); + if (extensionFunction != null) { + resolveIndirectArguments(actualArguments, extensionFunction); + if (profiling) { + activeProfilingFunctions.push(new ActiveFunction(awkName, tupleStartNanos)); + } + try { + push( + invokeExtension( + extensionFunction, + actualArguments, + position.lineNumber(), + true)); + } finally { + if (profiling) { + recordFunctionExit(System.nanoTime()); + } + } + position.next(); + return; + } + throw new AwkRuntimeException( + position.lineNumber(), + "function `" + qualifiedName + "' is not defined"); + } + private void execPrint(CountTuple tuple) throws IOException { long numArgs = tuple.getCount(); jrt.printDefault(numArgs == 0 ? new Object[] { jrt.jrtGetInputField(0) } : popArguments(numArgs)); diff --git a/src/test/java/io/jawk/backend/AVMExecuteTuplesSizeTest.java b/src/test/java/io/jawk/backend/AVMExecuteTuplesSizeTest.java new file mode 100644 index 00000000..a64c1839 --- /dev/null +++ b/src/test/java/io/jawk/backend/AVMExecuteTuplesSizeTest.java @@ -0,0 +1,225 @@ +package io.jawk.backend; + +/*- + * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ + * Jawk + * ჻჻჻჻჻჻ + * Copyright (C) 2006 - 2026 MetricsHub + * ჻჻჻჻჻჻ + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ + */ + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; + +/** + * Guards {@link AVM#executeTuples} against HotSpot's huge-method cliff. + *

+ * HotSpot never JIT-compiles a method whose bytecode is larger than + * {@code -XX:HugeMethodLimit} (8000, product build, not adjustable without + * a debug VM). If the interpreter dispatch loop crosses that limit, every + * AWK script runs ~4x slower, silently: no error, no warning, no JIT warmup. + * This has happened twice (see issue #562); different compilers sit at + * different sizes (ECJ output is ~40 bytecodes larger than javac's), so the + * threshold here keeps a comfortable safety margin below the real limit. + *

+ */ +public class AVMExecuteTuplesSizeTest { + + /** Guarded method name. */ + private static final String METHOD_NAME = "executeTuples"; + + /** + * Maximum allowed bytecode size: well under HotSpot's HugeMethodLimit + * (8000) so that neither javac nor ECJ output ever gets close to the + * cliff. If this test fails, extract opcode handlers from + * {@code executeTuples} into private methods (see the exec* helpers). + */ + private static final int MAX_CODE_LENGTH = 7500; + + @Test + public void testExecuteTuplesStaysUnderHugeMethodLimit() throws IOException { + Map codeLengths = readMethodCodeLengths(AVM.class); + Integer size = codeLengths.get(METHOD_NAME); + if (size == null) { + fail("Method " + METHOD_NAME + " not found in AVM.class - update this test if it was renamed"); + } + assertTrue( + METHOD_NAME + " is " + size + " bytecodes; it must stay <= " + MAX_CODE_LENGTH + + " or HotSpot (HugeMethodLimit=8000) will never JIT-compile the interpreter loop." + + " Extract opcode handlers into private exec* methods to shrink it.", + size <= MAX_CODE_LENGTH); + } + + @Test + public void testParserSeesPlausibleMethodSizes() throws IOException { + // Sanity-check the class-file parser itself: a tiny accessor must + // exist and be far smaller than the dispatch loop. + Map codeLengths = readMethodCodeLengths(AVM.class); + assertFalse("no methods with Code attributes found", codeLengths.isEmpty()); + Integer size = codeLengths.get(METHOD_NAME); + assertTrue( + "executeTuples should be the kind of method this guard exists for (>1000 bytecodes)", + size != null && size > 1000); + } + + /** + * Parses a class file and returns the {@code Code} attribute length of + * each method, keyed by method name. When a name is overloaded, the + * largest variant wins: the guard cares about the biggest body. + * + * @param clazz the class whose bytecode to inspect + * @return map of method name to bytecode ({@code code_length}) size + * @throws IOException if the class file cannot be read + */ + private static Map readMethodCodeLengths(Class clazz) throws IOException { + String resource = "/" + clazz.getName().replace('.', '/') + ".class"; + try (InputStream is = clazz.getResourceAsStream(resource)) { + if (is == null) { + throw new IOException("Cannot load " + resource); + } + DataInputStream in = new DataInputStream(is); + if (in.readInt() != 0xCAFEBABE) { + throw new IOException("Not a class file: " + resource); + } + in.readUnsignedShort(); // minor + in.readUnsignedShort(); // major + + // Constant pool: we only need the UTF-8 entries (method and + // attribute names); everything else is skipped by tag size. + int cpCount = in.readUnsignedShort(); + String[] utf8 = new String[cpCount]; + for (int i = 1; i < cpCount; i++) { + int tag = in.readUnsignedByte(); + switch (tag) { + case 1: // CONSTANT_Utf8 + utf8[i] = in.readUTF(); + break; + case 7: // Class + case 8: // String + case 16: // MethodType + case 19: // Module + case 20: // Package + skipFully(in, 2); + break; + case 15: // MethodHandle + skipFully(in, 3); + break; + case 3: // Integer + case 4: // Float + case 9: // Fieldref + case 10: // Methodref + case 11: // InterfaceMethodref + case 12: // NameAndType + case 17: // Dynamic + case 18: // InvokeDynamic + skipFully(in, 4); + break; + case 5: // Long + case 6: // Double + skipFully(in, 8); + i++; // longs and doubles take two constant pool slots + break; + default: + throw new IOException("Unknown constant pool tag " + tag + " in " + resource); + } + } + + skipFully(in, 6); // access_flags, this_class, super_class + int interfaceCount = in.readUnsignedShort(); + skipFully(in, 2 * interfaceCount); + + skipFieldsOrMethods(in, in.readUnsignedShort(), null, utf8); // fields + + Map codeLengths = new HashMap<>(); + skipFieldsOrMethods(in, in.readUnsignedShort(), codeLengths, utf8); // methods + return codeLengths; + } + } + + /** + * Reads a field_info/method_info table. When {@code codeLengths} is + * non-null, records each method's {@code Code} attribute length. + * + * @param in input positioned at the start of the table + * @param count number of entries + * @param codeLengths collector for method code lengths, or {@code null} + * to skip entries entirely (fields) + * @param utf8 constant pool UTF-8 entries + * @throws IOException if the class file cannot be read + */ + private static void skipFieldsOrMethods( + DataInputStream in, + int count, + Map codeLengths, + String[] utf8) + throws IOException { + for (int i = 0; i < count; i++) { + skipFully(in, 2); // access_flags + int nameIndex = in.readUnsignedShort(); + skipFully(in, 2); // descriptor_index + int attributeCount = in.readUnsignedShort(); + for (int a = 0; a < attributeCount; a++) { + int attrNameIndex = in.readUnsignedShort(); + int attrLength = in.readInt(); + if (codeLengths != null && "Code".equals(utf8[attrNameIndex])) { + skipFully(in, 4); // max_stack, max_locals + int codeLength = in.readInt(); + String name = utf8[nameIndex]; + Integer previous = codeLengths.get(name); + if (previous == null || previous < codeLength) { + codeLengths.put(name, codeLength); + } + // 8 bytes consumed so far: max_stack, max_locals, code_length + skipFully(in, attrLength - 8L); + } else { + skipFully(in, attrLength); + } + } + } + } + + /** + * Skips exactly {@code n} bytes, looping because + * {@link java.io.InputStream#skip(long)} may skip fewer. + * + * @param in the stream to skip in + * @param n number of bytes to skip + * @throws IOException if the end of the stream is reached first + */ + private static void skipFully(DataInputStream in, long n) throws IOException { + long remaining = n; + while (remaining > 0) { + long skipped = in.skip(remaining); + if (skipped <= 0) { + if (in.read() < 0) { + throw new IOException("Unexpected end of class file"); + } + skipped = 1; + } + remaining -= skipped; + } + } +}