{ "schema": "wasmc-tpc08-independent-evaluation-summary/v1", "provenance": "Curated transcription of measurements executed through awmcp on tpc08. Not a copy of every original raw sample or log. Native/core/library reports were run sequentially in separate phases.", "date": "2026-10-09", "repository": "cbgroom/wasmcrelease", "version": "0.0.21", "commit": "c888af39c11640644591ff42aeaf8b09f632f651", "compiler_sha256": "42c27913120d4ecf8a05ed990287767296aba9ae3b5af0a4e7ede182d5ad998c", "compiler_bytes": 1510219, "environment": { "host": "tpc08", "cpu": "13th Gen Intel(R) Core(TM) i7-13700", "logical_cpus": 24, "kernel": "6.18.33.2-microsoft-standard-WSL2", "architecture": "x86_64", "node": "v24.21.0", "v8": "13.6.233.17-node.53", "system_node": "v18.19.1", "c_compiler": "Ubuntu clang version 18.1.3 (1ubuntu1)", "c_flags": "-O3 -ffp-contract=off -std=c11", "cgroup_cpu_max": "max 100000", "cgroup_memory_max": "max", "cpu_affinity_pinned": false, "exclusive_host": false, "os_page_cache_flushed": false }, "methodology": { "compile": "One resident createCompiler() instance; ten warmups per corpus case, 31 batches. Batch size 50 below 1000 source bytes, 10 below 10000 bytes, otherwise 1. Batch-normalized per-operation quantiles; not request-tail latency. All five output digests matched the frozen manifest. 101 changing-source variants were also executed and checked.", "kernel": "Three independently authored import-free kernels. 100 small input oracles each; 8 warmups; 11 samples of 20M iterations. JS/Wasm ordering alternated, seeds varied. C/AOT measured later in their own phase; do not infer small C-vs-V8 speed differences across phases.", "native": "C and AOT: 11 process observations. Wasmi CLI: 5 observations, including process start and compilation. Native cold scalar: 20 process observations. AOT is the Wasmtime deployment path, not a direct WAsmC-to-native backend.", "library": "15 adaptive batches. JSON includes UTF8 input copy plus returned UTF8 decode, excludes initial TextEncoder. Host baseline is JSON.stringify(JSON.parse(text)). gzip compares defaults with Node zlib and records different output sizes. Base64 resident includes result allocation/drop but excludes host data transfer. Base64 end-to-end uses per-byte checked resource calls; it is not an optimized bulk transfer upper bound." }, "compilation": [ { "case": "small_scalar", "source_bytes": 126, "wasm_bytes": 44, "import_count": 0, "warm_p50_ms": 0.01532384, "warm_p95_ms": 0.03442672, "warm_min_ms": 0.00888078, "cli_build_p50_ms": 7.097, "cli_build_p95_ms": 9.792, "aot_build_cache_miss_p50_ms": 28.2, "aot_build_cache_hit_p50_ms": 65.529 }, { "case": "structured_variant", "source_bytes": 697, "wasm_bytes": 208, "import_count": 1, "warm_p50_ms": 0.0639748, "warm_p95_ms": 0.08807156, "warm_min_ms": 0.05569666, "cli_build_p50_ms": 8.514, "cli_build_p95_ms": 18.563, "aot_build_cache_miss_p50_ms": 71.468, "aot_build_cache_hit_p50_ms": 79.301 }, { "case": "policy_small", "source_bytes": 2562, "wasm_bytes": 733, "import_count": 7, "warm_p50_ms": 0.1383768, "warm_p95_ms": 0.2276724, "warm_min_ms": 0.1305531, "cli_build_p50_ms": 9.337, "cli_build_p95_ms": 20.23, "aot_build_cache_miss_p50_ms": 56.246, "aot_build_cache_hit_p50_ms": 49.125 }, { "case": "policy_large", "source_bytes": 30995, "wasm_bytes": 9454, "import_count": 14, "warm_p50_ms": 1.597142, "warm_p95_ms": 2.595195, "warm_min_ms": 1.435227, "cli_build_p50_ms": 13.681, "cli_build_p95_ms": 18.208, "aot_build_cache_miss_p50_ms": 81.58, "aot_build_cache_hit_p50_ms": 48.361 }, { "case": "std_mix_large", "source_bytes": 112300, "wasm_bytes": 31242, "import_count": 36, "warm_p50_ms": 5.141007, "warm_p95_ms": 6.162371, "warm_min_ms": 4.452532, "cli_build_p50_ms": 20.691, "cli_build_p95_ms": 26.487, "aot_build_cache_miss_p50_ms": 130.482, "aot_build_cache_hit_p50_ms": 52.35 } ], "fresh_node_process": { "n": 11, "p50": { "import_ms": 15.129988, "bootstrap_ms": 2.154718, "compile_ms": 6.338741, "module_ms": 0.073717, "instance_ms": 0.021518, "process_ms": 58.450266 }, "samples": { "import_ms": [ 13.363983, 15.003303, 14.999373, 15.129988, 14.820955, 15.149415, 15.347578, 15.083715, 15.758401, 20.57634, 16.823923 ], "bootstrap_ms": [ 2.602842, 1.756144, 1.879395, 1.925009, 2.154718, 1.840846, 1.801026, 5.656076, 2.639025, 3.159595, 2.329669 ], "compile_ms": [ 5.424078, 5.237195, 5.730825, 5.781571, 7.167437, 6.338741, 7.695958, 5.372392, 7.028202, 9.253037, 6.578783 ], "module_ms": [ 0.062197, 0.054539, 0.064701, 0.072834, 0.07905, 0.073717, 0.076571, 0.060841, 0.07968, 0.101054, 0.090221 ], "instance_ms": [ 0.01648, 0.019424, 0.021518, 0.021443, 0.025668, 0.019117, 0.023728, 0.015628, 0.028615, 0.034466, 0.024863 ], "process_ms": [ 54.021987, 58.661462, 57.705249, 58.450266, 53.952784, 56.536623, 59.089484, 55.309218, 66.497494, 99.783196, 68.592552 ] }, "scope": "Fresh Node test process with warm/uncontrolled OS caches; includes the test harness. Component medians must not be summed as a measured total." }, "cold_scalar_native": { "wasmi_cli_process_p50_ms": 7.955, "wasmi_cli_process_p95_ms": 17.085, "aot_process_p50_ms": 1.971, "aot_process_p95_ms": 3.661, "cli_bytes": 12374560, "aot_executable_bytes": 12388228, "c_reference_executable_bytes": 16432 }, "cache_interleaved_recheck": { "n_per_condition": 15, "hit_p50_ms": 43.639727, "hit_p95_ms": 79.604536, "miss_p50_ms": 35.944574, "miss_p95_ms": 59.450269, "interpretation": "No end-to-end latency benefit demonstrated for this tiny case. Cause not isolated." }, "runtime": [ { "case": "xorshift32", "iterations": 20000000, "wasm_bytes": 101, "wasm_kernel_p50_ms": 24.415326, "js_kernel_p50_ms": 24.54893, "c_kernel_p50_ms": 27.363384962, "c_process_p50_ms": 29.368388, "aot_process_p50_ms": 29.896101, "wasmi_cli_process_p50_ms": 131.513324 }, { "case": "branch_mix", "iterations": 20000000, "wasm_bytes": 140, "wasm_kernel_p50_ms": 17.526232, "js_kernel_p50_ms": 17.897366, "c_kernel_p50_ms": 18.387989998, "c_process_p50_ms": 20.666847, "aot_process_p50_ms": 20.72236, "wasmi_cli_process_p50_ms": 141.040699 }, { "case": "f64_recurrence", "iterations": 20000000, "wasm_bytes": 97, "wasm_kernel_p50_ms": 29.872897, "js_kernel_p50_ms": 29.94487, "c_kernel_p50_ms": 29.384340048, "c_process_p50_ms": 31.345475, "aot_process_p50_ms": 31.926115, "wasmi_cli_process_p50_ms": 59.807377 } ], "libraries": [ { "case": "json_compact", "input_bytes": 695, "wasm_p50_ms": 0.016087889534883715, "host_p50_ms": 0.005283747663551409 }, { "case": "json_compact", "input_bytes": 11727, "wasm_p50_ms": 0.21327180000000018, "host_p50_ms": 0.07877078260869555 }, { "case": "json_compact", "input_bytes": 37114, "wasm_p50_ms": 0.759746444444444, "host_p50_ms": 0.31999452272727197 }, { "case": "gzip_compress", "input_bytes": 4096, "wasm_p50_ms": 0.02818047107438017, "host_p50_ms": 0.028230068376068503, "wasm_output_bytes": 389, "host_output_bytes": 382 }, { "case": "gzip_compress", "input_bytes": 65536, "wasm_p50_ms": 0.23045446551723922, "host_p50_ms": 0.19849963636363555, "wasm_output_bytes": 1197, "host_output_bytes": 1344 }, { "case": "base64_encode", "input_bytes": 64, "resident_p50_ms": 0.0018824568171854164, "per_byte_driver_end_to_end_p50_ms": 0.009775270270269688, "host_p50_ms": 0.00023965678449263306 } ], "functional": { "selected_repository_commands": 40, "passed": 24, "nonzero_commands": 16, "classification_note": "Maintainer refresh-receipt preconditions and missing development tools prevented these commands from entering their intended tests. This is not a 40% product failure rate. See original per-command logs for exact prerequisites.", "compiler_regression": { "frozen_outputs": 35, "execution_oracles": 7, "cli_outputs": 25 }, "additional_validation": { "frozen_outputs": 30, "expression_cases": 23, "managed_repeated_calls": 192 }, "managed_api": { "collection_calls": 768, "boolean_conditional_calls": 1536, "numeric_bool_rejections": 6 }, "std": { "api_inventory": 73, "consumer_count": 3, "consumer_calls": 7680, "coverage_note": "API inventory count is not proof of full semantic coverage of all 73 operations." }, "library_inventory": { "packages": 42, "api_routes": 236, "index_entries": 278, "core_artifacts_validated": 22, "other_native_adapter_artifacts": 6, "source_only_roots": 14, "full_device_matrix_executed": false }, "json_checks": 25, "gzip_checks": 13, "u64_differential_checks": 304, "json_limits": { "input_65536_bytes": "accepted", "input_65537_bytes": "input-too-large", "depth_64": "accepted", "depth_65": "depth-limit", "pointers_65": "too-many-pointers" }, "gzip_limits": { "input_1_MiB": "accepted", "input_1_MiB_plus_1": "input-too-large", "output_4_MiB": "accepted", "output_4_MiB_plus_1": "output-too-large" } }, "memory_observations": { "resident_compiler": { "compilations": 10000, "linear_memory_before_bytes": 1310720, "linear_memory_after_bytes": 1310720 }, "base64_small_input": { "calls": 10000, "explicit_drops": 20000, "linear_memory_before_bytes": 3670016, "linear_memory_after_bytes": 3670016 }, "scope": "Linear memory high-water observations only, not RSS, heap leak proof, or long-running service qualification." }, "findings": [ { "id": "F1", "title": "Core frontend reports success for invalid Wasm", "source": "package independent:bool_f64; interface api {run:func()->bool{return 1.5;}} world app {export api;}", "core_compile_detailed_ok": true, "wasm_bytes": 43, "webassembly_validate": false, "validation_error": "type error in fallthru[0] (expected i32, got f64) @+42", "native_build_exit_code": 0, "native_output_valid": false, "managed_with_real_root_rejects": true, "also_reproduced_with_s32_return": true, "security_scope": "The engine rejects invalid code; this is not evidence of a sandbox escape." }, { "id": "F2", "title": "Core bool lane differs from managed source validation", "core_bool_return_2": 2, "core_bool_return_negative_1": -1, "managed_with_real_root_rejects_both": true }, { "id": "F3", "title": "Std Base64 trap with specific byte-buffer inputs", "provider": "4.9.0", "std": "1.4.1", "integration": "Same Core provider/Std export pattern as the public base64 example; fresh instance for each input length. Input byte i is i & 255; all input pushes succeeded.", "passed_lengths": [ 64 ], "trapped_lengths": [ 256, 512, 1024, 2048, 3072, 4096, 8192, 65536 ], "error": "RuntimeError: unreachable", "failed_stage": "std_base64_try_encode_standard", "root_cause": "Not isolated. These probes vary both input length and byte content. Do not infer a specific size limit or UTF8 cause. Other SDK/Component paths were not tested." }, { "id": "F4", "title": "Node18 Std compatibility preflight rejection", "node": "v18.19.1", "missing_features": [ "function-references", "tail-call" ], "node24_passed": true } ], "limitations": [ "Only Linux x64 / WSL2 was executed, not all released targets or device capabilities.", "No security audit or source audit/rebuild of the private compiler implementation.", "All 236 APIs were not executed; catalogue closure/validation is not runtime coverage.", "C and V8 kernel results come from separate phases on a non-isolated host.", "Full maintainer refresh qualification was not reproduced. Some custom harness attempts stopped at newly observed errors; valid prefix measurements and later isolated reprobes are distinguished.", "Large managed-list benchmark was not completed because the initial harness used unsupported .get syntax for that list shape; this is not classified as a library failure." ], "original_remote_workspace": "/workspace/wasmc-eval-20261009", "original_remote_reports": [ "results/functional.json", "results/core-bench.json", "results/native-cli-perf.json", "results/supplement.json", "results/libs-bench.json", "results/type-probes.json", "results/base64-probes.json" ], "original_tracked_repository_status": "git status --short produced empty output after tests", "native_artifact_identity": { "archive_name": "wasmc-v0.0.21-x86_64-unknown-linux-gnu.tar.gz", "archive_sha256": "a990551897498091685a05fd309edcff79b0a7db728079b6e71470e369888b7b", "executable_sha256": "a6a5fc5f46a63373372d81b15a3ef4b81d36e1004292e69140341fc587debc33" } }