Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions backend/app/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,9 @@ def admin_credit(member_id:UUID,incoming:CreditGrant,actor=Depends(administrator

@app.get('/admin/evaluations')
def admin_jobs(actor=Depends(administrator)):
return [present(j) | {'user_id':j['user_id']} for j in db.list()[:200]]
rows = db.list()[:200]
metadata = db.get_presentations([j['id'] for j in rows])
return [present(j, metadata.get(j['id'], {})) | {'user_id':j['user_id']} for j in rows]

@app.get('/admin/evaluations/{job_id}/logs')
def admin_logs(job_id:UUID,actor=Depends(administrator)):
Expand Down Expand Up @@ -145,15 +147,17 @@ def worker(job_id: UUID, authorization: str | None = Header(default=None)):
if not job: raise HTTPException(404, 'Evaluation not found')
return job

def present(job):
job = {**job, 'allocation': db.get_presentation(job['id'], 'allocation')}
publication=db.get_presentation(job['id'],'publication')
def present(job, metadata=None):
if metadata is None:
metadata = db.get_presentations([job['id']]).get(job['id'], {})
job = {**job, 'allocation': metadata.get('allocation')}
publication=metadata.get('publication')
if job['state']=='succeeded':
desired=job['config'].get('visibility')=='public'
if desired and not publication:publication={'state':'pending'}
elif publication and publication.get('desired_public')!=desired:
publication={**publication,'state':'pending' if desired else 'unpublishing'}
summary=db.get_presentation(job['id'],'summary') or {}
summary=metadata.get('summary') or {}
return public_job(job) | {'publication':publication, 'omitted_count': summary.get('omitted_count',0)}

@app.get('/health')
Expand Down Expand Up @@ -240,11 +244,15 @@ def schema(): return EvaluationRequest.model_json_schema()

@app.get('/evaluations')
def evaluations(user_id=Depends(user)):
return [present(job) for job in db.list(user_id)]
rows = db.list(user_id)
metadata = db.get_presentations([j['id'] for j in rows])
return [present(job, metadata.get(job['id'], {})) for job in rows]

@app.get('/experiments')
def published():
return [present(j) for j in db.public_jobs() if db.get_presentation(j['id'], 'summary')]
rows = db.public_jobs()
metadata = db.get_presentations([j['id'] for j in rows])
return [present(j, metadata.get(j['id'], {})) for j in rows if metadata.get(j['id'], {}).get('summary')]

def readable(job_id, authorization):
job = db.get(str(job_id))
Expand Down
12 changes: 12 additions & 0 deletions backend/app/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,18 @@ def get_presentation(self, job_id, name):
with self.engine.connect() as c:
return c.execute(select(presentation.c.data).where(presentation.c.job_id == job_id, presentation.c.name == name)).scalar()

def get_presentations(self, job_ids):
if not job_ids:
return {}
with self.engine.connect() as c:
rows = c.execute(select(presentation).where(
presentation.c.job_id.in_(job_ids),
presentation.c.name.in_(('allocation', 'publication', 'summary')))).mappings()
result = {}
for row in rows:
result.setdefault(row['job_id'], {})[row['name']] = row['data']
return result

def visibility(self, job_id, value):
with self.engine.begin() as c:
row = c.execute(select(jobs).where(jobs.c.id == job_id).with_for_update()).mappings().one()
Expand Down
25 changes: 25 additions & 0 deletions backend/tests/test_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -386,3 +386,28 @@ def test_auto_cpu_respects_cpu_limit(service):
response=submit(client,payload(compute_type='gpu'))
assert response.status_code==403
assert 'max_cpu_count' in response.json()['detail']


def test_history_batches_metadata_and_keeps_owner_scope(service):
from sqlalchemy import event
_, db, client = service
first = submit(client).json()['id']
second = submit(client, key='two').json()['id']
foreign = submit(client, user=OTHER).json()['id']
db.put_presentation(first, 'summary', {'omitted_count': 3})
db.put_presentation(foreign, 'summary', {'omitted_count': 999})
statements = []
def capture(conn, cursor, statement, parameters, context, executemany):
if statement.lstrip().upper().startswith('SELECT') and 'va_presentation' in statement:
statements.append(statement)
event.listen(db.engine, 'before_cursor_execute', capture)
try:
response = client.get('/evaluations', headers={'Authorization': 'Bearer '+USER})
finally:
event.remove(db.engine, 'before_cursor_execute', capture)
assert response.status_code == 200
rows = {row['id']: row for row in response.json()}
assert set(rows) == {first, second}
assert rows[first]['omitted_count'] == 3
assert rows[second]['omitted_count'] == 0
assert len(statements) == 1
9 changes: 9 additions & 0 deletions next-src/src/app/pages.css
Original file line number Diff line number Diff line change
Expand Up @@ -1015,6 +1015,15 @@ body:has(.arena-home) .page-hills {
.eval-result-links > span { color: var(--text-muted); font-size: 12px; }
@media (max-width: 600px) { .eval-result-links { padding: 0 16px 16px; gap: 8px; } .eval-result-links > span { flex-basis: 100%; } }

.eval-success-section { margin: 24px 0 36px; }
.eval-success-scroll { overflow-x: auto; border: 1px solid var(--border); border-radius: 16px; }
.eval-success-table { width: 100%; border-collapse: collapse; text-align: left; font-size: 14px; }
.eval-success-table th, .eval-success-table td { padding: 16px 20px; border-bottom: 1px solid var(--border); vertical-align: middle; }
.eval-success-table thead th { color: var(--text-muted); font-size: 12px; font-weight: 600; white-space: nowrap; }
.eval-success-table tbody th { font-weight: 600; min-width: 150px; overflow-wrap: anywhere; }
.eval-success-table tbody tr:last-child > * { border-bottom: 0; }
.eval-success-actions { display: flex; gap: 14px; flex-wrap: wrap; }
.eval-success-actions a { white-space: nowrap; text-underline-offset: 4px; }
/* ---------- Evaluate: generation options ---------- */
.eval-generation {
margin: 8px 0 18px;
Expand Down
2 changes: 1 addition & 1 deletion next-src/src/components/EvaluationRunner.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ export function EvaluationRunner() {
<div className="eval-submit"><button className="button-primary" disabled={busy} aria-describedby={blockers.length ? "evaluation-blockers" : undefined}>{busy ? 'Preparing…' : 'Run evaluation →'}</button>{blockers.length>0 && <div id="evaluation-blockers" tabIndex={-1} aria-live="polite"><strong>Before you can run</strong><ul>{blockers.map((message,i)=><li key={i}>{message}</li>)}</ul></div>}{error && <p role="alert">{error}</p>}<small>Runs continue in the background until complete or cancelled.{limits.max_runtime_seconds!==null && ` Admin runtime limit: ${Math.round(limits.max_runtime_seconds/60)} minutes.`}{!ownKeys && limits.require_credits && " Runs also stop when their reserved execution credits are used; unused time is returned."}</small></div>
</aside>
</form>}
{tab === 'runs' && <section className="evaluation-jobs"><h2>Your evaluations</h2>{!runsLoaded && !jobs.length && !runsError && <div className="eval-history-loading" role="status" aria-live="polite" aria-busy="true"><Penguin size={48} state="loading" /><p>Loading your evaluations…</p></div>}{runsError && <div role="alert"><p>{runsError}</p><button type="button" className="button-secondary" disabled={runsRefreshing} onClick={() => setRunsRetry(n => n + 1)}>{runsRefreshing ? 'Retrying…' : 'Retry loading evaluations'}</button></div>}{runsLoaded && !jobs.length && !runsError && <p>No evaluations yet. Submitted runs will appear here.</p>}{jobs.map(job => { const open = openRuns[job.id] ?? isActive(job.state); const body = `run-body-${job.id}`; return <article className={`eval-job${open ? ' is-open' : ''}`} key={job.id}>
{tab === 'runs' && <section className="evaluation-jobs"><h2>Your evaluations</h2>{!runsLoaded && !jobs.length && !runsError && <div className="eval-history-loading" role="status" aria-live="polite" aria-busy="true"><Penguin size={48} state="loading" /><p>Loading your evaluations…</p></div>}{runsError && <div role="alert"><p>{runsError}</p><button type="button" className="button-secondary" disabled={runsRefreshing} onClick={() => setRunsRetry(n => n + 1)}>{runsRefreshing ? 'Retrying…' : 'Retry loading evaluations'}</button></div>}{runsLoaded && !jobs.length && !runsError && <p>No evaluations yet. Submitted runs will appear here.</p>}{jobs.some(job => job.state === 'succeeded') && <section className="eval-success-section" aria-labelledby="successful-runs-title"><h3 id="successful-runs-title">Successful runs</h3><div className="eval-success-scroll"><table className="eval-success-table"><thead><tr><th scope="col">Run</th><th scope="col">Constitution</th><th scope="col">Models</th><th scope="col">Scenarios</th><th scope="col">Results</th></tr></thead><tbody>{jobs.filter(job => job.state === 'succeeded').map(job => <tr key={job.id}><th scope="row">{job.name}</th><td>{job.constitution}</td><td>{job.models_count}</td><td>{job.scenario_count}</td><td><div className="eval-success-actions"><a href={`/run/?slug=account/${job.id}`}>View results →</a><a href={`/transcript/?run=account/${job.id}`}>Transcripts</a></div></td></tr>)}</tbody></table></div></section>}{jobs.length > 0 && <h3>All runs</h3>}{jobs.map(job => { const open = openRuns[job.id] ?? isActive(job.state); const body = `run-body-${job.id}`; return <article className={`eval-job${open ? ' is-open' : ''}`} key={job.id}>
{/* Active runs start open; finished ones stay collapsed until clicked */}
<button type="button" className="eval-job-toggle" aria-expanded={open} aria-controls={body} onClick={() => setOpenRuns(o => ({ ...o, [job.id]: !open }))}><span className="eval-state" data-state={job.state}>{job.state === 'succeeded' ? 'Completed' : label(job.state)}</span><span className="eval-job-title"><strong>{job.name}</strong><small>{job.constitution} · {job.models_count} models · {job.scenario_count} scenarios · {job.visibility} · {job.engine === 'inspect' ? 'Inspect' : 'Native'}</small></span><span className="eval-job-chevron" aria-hidden="true" /></button>
{job.state === 'succeeded' && <div className="eval-result-links"><a className="button-primary" href={`/run/?slug=account/${job.id}`}>View results →</a><a className="button-secondary" href={`/transcript/?run=account/${job.id}`}>Transcripts</a><span>Rankings and individual judgments</span></div>}
Expand Down
Loading