{ "date": "2026-10-13", "provider": "ollama", "model": "gemma4:12b-mlx", "context_budget": 32768, "notes": [ "Smoke tasks only; prompt revisions and concurrent server use prevent fair latency comparisons.", "Null protected-tests audit means only final contents were checked in that historical pilot.", "The native-no-thinking repair pilot temporarily changed and restored a test; passing final artifacts are not a clean constraint-adherence pass.", "The final-native-thinking trials audit protected tests after every tool result." ], "records": [ { "experiment": "native-no-thinking", "case": "repair", "artifact_tests_passed": true, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": null, "seconds": 35.84, "tool_calls": 18, "tool_names": [ "bash", "edit", "read", "write" ], "input_tokens": 33360, "output_tokens": 1213, "errors": [], "thinking": false, "temperature": 1.0, "tool_mode": "auto" }, { "experiment": "native-no-thinking", "case": "create", "artifact_tests_passed": true, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": null, "seconds": 43.95, "tool_calls": 9, "tool_names": [ "bash", "python", "read", "search", "write" ], "input_tokens": 20099, "output_tokens": 1604, "errors": [], "thinking": false, "temperature": 1.0, "tool_mode": "auto" }, { "experiment": "native-thinking", "case": "repair", "artifact_tests_passed": true, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": null, "seconds": 71.4, "tool_calls": 15, "tool_names": [ "bash", "edit", "read" ], "input_tokens": 48201, "output_tokens": 2764, "errors": [], "thinking": true, "temperature": 1.0, "tool_mode": "auto" }, { "experiment": "native-thinking", "case": "create", "artifact_tests_passed": true, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": null, "seconds": 175.48, "tool_calls": 11, "tool_names": [ "bash", "python", "read", "search", "write" ], "input_tokens": 28587, "output_tokens": 2073, "errors": [], "thinking": true, "temperature": 1.0, "tool_mode": "auto" }, { "experiment": "native-thinking-0.2", "case": "repair", "artifact_tests_passed": true, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": null, "seconds": 170.25, "tool_calls": 10, "tool_names": [ "bash", "edit", "read" ], "input_tokens": 25218, "output_tokens": 1451, "errors": [], "thinking": true, "temperature": 0.2, "tool_mode": "auto" }, { "experiment": "native-thinking-0.2", "case": "create", "artifact_tests_passed": true, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": null, "seconds": 60.8, "tool_calls": 16, "tool_names": [ "bash", "python", "read", "search", "write" ], "input_tokens": 45172, "output_tokens": 1787, "errors": [], "thinking": true, "temperature": 0.2, "tool_mode": "auto" }, { "experiment": "forced-json-thinking", "case": "repair", "artifact_tests_passed": false, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": null, "seconds": 180.23, "tool_calls": 0, "tool_names": [], "input_tokens": 0, "output_tokens": 0, "errors": [ { "error": "Model turn failed", "error_type": "ModelTurnError" } ], "thinking": true, "temperature": 1.0, "tool_mode": "json" }, { "experiment": "forced-json-thinking", "case": "create", "artifact_tests_passed": false, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": null, "seconds": 5.47, "tool_calls": 0, "tool_names": [], "input_tokens": 0, "output_tokens": 0, "errors": [ { "error": "Expecting value: line 1 column 1 (char 0)", "error_type": "ModelTurnError" } ], "thinking": true, "temperature": 1.0, "tool_mode": "json" }, { "experiment": "final-native-thinking", "case": "repair", "artifact_tests_passed": true, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": [], "seconds": 32.37, "tool_calls": 8, "tool_names": [ "bash", "edit", "read" ], "input_tokens": 18018, "output_tokens": 1047, "errors": [], "thinking": true, "temperature": 1.0, "tool_mode": "auto" }, { "experiment": "final-native-thinking", "case": "create", "artifact_tests_passed": true, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": [], "seconds": 52.74, "tool_calls": 10, "tool_names": [ "bash", "python", "read", "search", "write" ], "input_tokens": 28848, "output_tokens": 1885, "errors": [], "thinking": true, "temperature": 1.0, "tool_mode": "auto" }, { "experiment": "syntax-recovery-native-thinking", "case": "create", "artifact_tests_passed": true, "tests_unchanged_at_finish": true, "protected_tests_changed_during_run": [], "seconds": 47.82, "tool_calls": 11, "tool_names": [ "bash", "python", "read", "search", "write" ], "input_tokens": 29912, "output_tokens": 1853, "errors": [], "thinking": true, "temperature": 1.0, "tool_mode": "auto" } ], "local_gemma4_trials": { "date": "2026-10-13", "host": { "hardware": "Apple M4 Max, 40-core GPU, 128 GB unified memory", "dedicated_gpu_vram_gb": 0 }, "runtime_versions": { "ollama": "0.32.7", "lm_studio": "0.4.16+1 / CLI 10360", "llama_cpp": "10360 (48d22e295)" }, "context_tokens": 32768, "model_artifacts": [ { "model": "Gemma 4 26B-A4B-it QAT Q4_0 GGUF", "repo": "google/gemma-4-26B-A4B-it-qat-q4_0-gguf", "revision": "d1c082be9cf3c8a514acf63b8761f4b41935842e", "bytes": 14439363584, "sha256": "3eca3b8f6d7baf218a7dd6bba5fb59a56ee25fe2d567b6f5f589b4f697eca51d" }, { "model": "EmbeddingGemma 2 text BF16 GGUF", "repo": "ggml-org/embeddinggemma-2-GGUF", "revision": "bfcd298762cc34d0357ece5ebdd31791a3a374d8", "bytes": 557950176, "sha256": "68bae29d62fb8c7d23e98d21fd4662753ddd636e6b62b8a70dfe059a9844f216" } ], "gemma_12b_baseline": { "provider": "ollama", "model": "gemma4:12b-mlx", "cases": [ { "name": "repair", "passed": true, "seconds": 54.94, "tool_calls": [ "bash", "read", "read", "bash", "read", "bash", "edit", "read", "python", "edit", "bash" ], "tests_unchanged": true, "protected_tests_changed": [], "errors": [] }, { "name": "create", "passed": true, "seconds": 40.09, "tool_calls": [ "search", "read", "write", "python", "python", "python", "bash" ], "tests_unchanged": true, "protected_tests_changed": [], "errors": [] } ] }, "gemma_26b": [ { "provider": "ollama", "model": "gemma4-codegem-26b-q4", "cases": [ { "name": "repair", "passed": true, "seconds": 19.62, "tool_calls": [ "bash", "read", "read", "bash", "edit", "bash", "read" ], "tests_unchanged": true, "protected_tests_changed": [], "errors": [] }, { "name": "create", "passed": true, "seconds": 16.48, "tool_calls": [ "search", "search", "bash", "read", "python", "write", "python", "bash" ], "tests_unchanged": true, "protected_tests_changed": [], "errors": [] } ] }, { "provider": "lmstudio", "model": "gemma4-codegem-26b-q4", "cases": [ { "name": "repair", "passed": true, "seconds": 5.67, "tool_calls": [ "bash", "read", "read", "edit", "bash" ], "tests_unchanged": true, "protected_tests_changed": [], "errors": [] }, { "name": "create", "passed": true, "seconds": 10.4, "tool_calls": [ "search", "bash", "read", "write", "python", "bash" ], "tests_unchanged": true, "protected_tests_changed": [], "errors": [] } ] }, { "provider": "llamacpp", "model": "gemma4-codegem-26b-q4", "cases": [ { "name": "repair", "passed": true, "seconds": 17.5, "tool_calls": [ "bash", "read", "read", "bash", "edit", "bash" ], "tests_unchanged": true, "protected_tests_changed": [], "errors": [] }, { "name": "create", "passed": true, "seconds": 16.98, "tool_calls": [ "search", "read", "write", "python", "bash" ], "tests_unchanged": true, "protected_tests_changed": [], "errors": [] } ] } ], "lm_studio_auto_fallback": "Direct imported GGUF native tools produced an explicit Jinja template error. CodeGem retried with strict JSON Schema tool responses; both isolated tasks passed.", "embedding_runtime_checks": [ { "runtime": "llama.cpp 10360", "result": "failed to load GGUF: unknown model architecture gemma-embedding2" }, { "runtime": "Ollama 0.32.7", "result": "pull refused: model requires a newer Ollama version" }, { "runtime": "Sentence Transformers 6.1.0 / Transformers 5.19.0 / PyTorch 2.10.0 / torchvision 0.25.0", "model": "google/embeddinggemma-2", "device": "Apple MPS", "result": "loaded official Hugging Face weights and completed local embedding/index/search run" } ], "caveats": [ "Coding trials are two tiny isolated smoke tasks, not a held-out coding benchmark.", "Cross-runtime timings are not a controlled throughput comparison.", "This Apple unified-memory host does not validate dedicated 32 GB VRAM behavior.", "Live EmbeddingGemma results are from six labeled source-file queries; they demonstrate local operation, not general retrieval quality." ], "embeddinggemma2_live_retrieval": { "provider": "sentence-transformers", "model": "google/embeddinggemma-2", "device": "mps", "dimensions": 256, "indexed_chunks": 1789, "embedded_on_incremental_validation": 16, "reused_on_incremental_validation": 1773, "recall_at_5": 1.0, "mrr_at_5": 1.0, "queries": 6, "scope": [ "src/superqode/codegem", "src/superqode/harness" ], "miss": null, "note": "small hand-labeled regression smoke test only; not a general retrieval benchmark", "offline_only_verified": true, "query_note": "natural user queries; a more formal adapter-resume phrasing initially ranked that file sixth, while \u201chow does CodeGem resume an existing session\u201d ranked it first" }, "embedding_runtime_failures": [ "Ollama 0.32.7: EmbeddingGemma 2 package requires a newer server.", "llama.cpp 10360: GGUF architecture gemma-embedding2 is unknown.", "Sentence Transformers base install omitted Pillow and torchvision, which the Gemma 4 processor imports for text-only input; CodeGem semantic-local now includes these dependencies.", "On this host, MPS access from the restricted execution sandbox falsely reports an unsupported OS. The host-level MPS inference and retrieval run succeeded." ] } }