harbor-framework/terminal-bench

diff --git a/tools/rubric-regression/test_generate_valsmith.py b/tools/rubric-regression/test_generate_valsmith.pynew file mode 100644index 0000000..195339a--- /dev/null+++ b/tools/rubric-regression/test_generate_valsmith.py@@ -0,0 +1,85 @@+import importlib.util+import json+from pathlib import Path+++MODULE_PATH = Path(__file__).with_name("generate.py")+++def load_generator():+    spec = importlib.util.spec_from_file_location("rubric_regression_generate", MODULE_PATH)+    module = importlib.util.module_from_spec(spec)+    spec.loader.exec_module(module)+    return module+++def test_select_tasks_reports_unlabelled_and_malformed_entries(tmp_path, monkeypatch):+    generator = load_generator()+    for name in ("fail-rubric-alpha", "fail-rubric-beta", "fail-rubric-gamma"):+        (tmp_path / name).mkdir()+    monkeypatch.setattr(generator, "TEST_TASKS_DIR", tmp_path)+    labels = {+        "tasks": {+            "alpha": {"path": "fail-rubric-alpha", "labels": {"criterion_a": "fail"}},+            "beta": {"path": "fail-rubric-beta", "labels": {"criterion_b": "pass"}},+        }+    }++    selected, problems = generator.select_tasks(labels)++    assert selected == {"fail-rubric-alpha": "criterion_a"}+    assert problems == [+        "fail-rubric-beta: expected exactly one 'fail' label, got {'criterion_b': 'pass'}",+        "fail-rubric-gamma: no labels.json entry",+    ]+++def test_generate_task_stages_payload_and_renders_oracle(tmp_path, monkeypatch):+    generator = load_generator()+    source = tmp_path / "fail-rubric-example"+    source.mkdir()+    (source / "instruction.md").write_text("payload")+    (source / "nested").mkdir()+    (source / "nested" / "extra.txt").write_text("preserved")++    templates = tmp_path / "templates"+    templates.mkdir()+    templates_data = {+        "Dockerfile": "FROM python:3.12\n",+        "instruction.md": "Review the task.\n",+        "task.toml": "version = '1'\n",+        "test.sh": "#!/bin/sh\n",+        "test_state.py": 'PLANTED = "__PLANTED_CRITERION__"\n',+        "solve.sh": "#!/bin/sh\nprintf '%s' '__VERDICTS_JSON__' > /app/verdicts.json\n",+    }+    for name, contents in templates_data.items():+        (templates / name).write_text(contents)+    rubric = tmp_path / "rubric.toml"+    rubric.write_text("rubric contents")+    monkeypatch.setattr(generator, "TEMPLATES_DIR", templates)+    monkeypatch.setattr(generator, "RUBRIC_PATH", rubric)++    output = tmp_path / "output"+    generator.generate_task(source, output, "criterion_b", ["criterion_a", "criterion_b"])++    staged = output / "environment" / "task-under-review" / source.name+    assert (staged / "instruction.md").read_text() == "payload"+    assert (staged / "nested" / "extra.txt").read_text() == "preserved"+    assert (output / "environment" / "rubric.toml").read_text() == "rubric contents"+    assert (output / "tests" / "test_state.py").read_text() == 'PLANTED = "criterion_b"\n'+    solve = output / "solution" / "solve.sh"+    assert solve.stat().st_mode & 0o111+    rendered = solve.read_text()+    oracle = json.loads(rendered.split("printf '%s' '", 1)[1].split("' >", 1)[0])+    assert oracle["checks"]["criterion_a"]["outcome"] == "pass"+    assert oracle["checks"]["criterion_b"]["outcome"] == "fail"+++def test_oracle_verdicts_marks_only_the_planted_criterion_failed():+    generator = load_generator()+    verdicts = json.loads(generator.oracle_verdicts(["one", "two", "three"], "two"))+    assert {name: check["outcome"] for name, check in verdicts["checks"].items()} == {+        "one": "pass",+        "two": "fail",+        "three": "pass",+    }