148 lines
8.4 KiB
Python
148 lines
8.4 KiB
Python
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
|
|
spec=importlib.util.spec_from_file_location('dashboard',Path(__file__).resolve().parents[1]/'server.py')
|
|
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
|
|
|
class DashboardTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp=tempfile.TemporaryDirectory(prefix='dashboard-qa-')
|
|
self.old=(m.DATA,m.LOGS)
|
|
m.DATA=Path(self.temp.name)/'state';m.LOGS=Path(self.temp.name)/'logs';m.LOGS.mkdir()
|
|
m.initialize()
|
|
m.LATEST.clear();m.LATEST.update({'ts':time.time(),'boot_id':'test-current','monotonic_s':1000,'issues':[]})
|
|
|
|
def tearDown(self):
|
|
m.DATA,m.LOGS=self.old;self.temp.cleanup()
|
|
|
|
def test_import_replay_and_clock_adjustment(self):
|
|
now=int(time.time())
|
|
row={'observed_at':m.iso(now),'first_at':m.iso(now+3600),'last_at':m.iso(now-3600),
|
|
'first_monotonic_us':'11000000','last_monotonic_us':'487000000',
|
|
'boot_id':'oldboot','source':'kernel','category':'power','severity':'ERROR','count':93,
|
|
'device':'0000:84:00.0','detail':'GPU does not have the necessary power cables connected.'}
|
|
p=m.LOGS/'events-test.jsonl';p.write_text(json.dumps(row)+'\n'+json.dumps(row)+'\n')
|
|
with m.connect() as db:
|
|
m.import_logs(db);m.import_logs(db)
|
|
self.assertEqual(db.execute('SELECT COUNT(*) FROM events').fetchone()[0],1)
|
|
result=m.events({'days':['7']})
|
|
self.assertEqual(result['total'],1)
|
|
self.assertEqual(result['items'][0]['state'],'history')
|
|
self.assertAlmostEqual(result['items'][0]['first_ts'],now-3600-476,delta=1)
|
|
self.assertTrue(result['items'][0]['time_note'])
|
|
self.assertEqual(result['items'][0]['raw']['first_at'],row['first_at'])
|
|
|
|
def test_history_does_not_invent_old_cpu_utilization(self):
|
|
now=int(time.time())-40
|
|
with m.connect() as db:
|
|
db.execute('INSERT INTO metrics VALUES(?,NULL,NULL,NULL,NULL,NULL,NULL,NULL,?,?)',
|
|
(now-1500,json.dumps([{'uuid':'gpu-a','name':'P100','temperature_c':40,'power_w':30}]),'legacy'))
|
|
for offset,cpu in [(0,40),(10,80)]:
|
|
db.execute('INSERT INTO metrics VALUES(?,?,?,?,?,?,?,?,?,?)',
|
|
(now+offset,cpu,60,45,1,2,3,4,json.dumps([{'uuid':'gpu-a','name':'P100','utilization':90,'temperature_c':50,'power_w':100}]),'live'))
|
|
data=m.history('1h')
|
|
legacy=[p for p in data['points'] if not p['live']]
|
|
self.assertTrue(legacy)
|
|
self.assertIsNone(legacy[0]['cpu'])
|
|
self.assertIsNone(legacy[0]['gpus']['gpu-a']['utilization'])
|
|
self.assertEqual(legacy[0]['gpus']['gpu-a']['temperature_c'],40)
|
|
self.assertEqual(data['devices']['gpu-a'],'P100')
|
|
self.assertTrue(any(p['cpu'] is not None for p in data['points']))
|
|
|
|
def test_filters_and_partial_tail(self):
|
|
now=int(time.time())
|
|
row={'observed_at':m.iso(now),'category':'storage','severity':'ERROR','detail':'nvme timeout'}
|
|
p=m.LOGS/'events-tail.jsonl';p.write_text(json.dumps(row))
|
|
with m.connect() as db:
|
|
m.import_logs(db)
|
|
self.assertEqual(db.execute('SELECT COUNT(*) FROM events').fetchone()[0],0)
|
|
with p.open('a') as f:f.write('\n')
|
|
with m.connect() as db:m.import_logs(db)
|
|
self.assertEqual(m.events({'category':['storage']})['total'],1)
|
|
self.assertEqual(m.events({'category':['power']})['total'],0)
|
|
self.assertEqual(m.events({'q':['timeout']})['total'],1)
|
|
|
|
def test_stale_status_is_unknown_not_recovered(self):
|
|
m.LATEST.update({'ts':time.time()-300})
|
|
now=int(time.time())
|
|
row={'observed_at':m.iso(now),'category':'power','severity':'ERROR','detail':'power error'}
|
|
(m.LOGS/'events-stale.jsonl').write_text(json.dumps(row)+'\n')
|
|
with m.connect() as db:m.import_logs(db)
|
|
self.assertEqual(m.events({})['items'][0]['state'],'unknown')
|
|
|
|
def test_link_reports_merge_and_keep_two_cards_separate(self):
|
|
now=int(time.time())
|
|
a={'id':'gpu-a','label':'第 1 张 P100','pci':'0000:03:00.0'}
|
|
b={'id':'gpu-b','label':'第 2 张 P100','pci':'0000:84:00.0'}
|
|
m.LATEST.update({'device_map':{'0000:00:03.0':a,'0000:03:00.0':a,'0000:80:03.0':b},
|
|
'gpus':[{'uuid':'gpu-a'},{'uuid':'gpu-b'}],'expected_gpu_count':2})
|
|
def row(device,detail,timestamp,mono):
|
|
return {'observed_at':m.iso(now),'first_at':m.iso(timestamp),'last_at':m.iso(timestamp),
|
|
'first_monotonic_us':str(mono),'last_monotonic_us':str(mono),
|
|
'boot_id':'test-current','source':'kernel','category':'pcie_correctable','severity':'WARN',
|
|
'device':device,'count':1,'detail':detail}
|
|
rows=[row('0000:00:03.0','PCIe Bus Error: severity=Correctable',now-50,950000000),
|
|
row('0000:00:03.0','[6] BadTLP',now-50,950000001),
|
|
row('0000:03:00.0','PCIe Bus Error: severity=Correctable',now-50,950000002),
|
|
row('0000:80:03.0','PCIe Bus Error: severity=Correctable',now-40,960000000)]
|
|
path=m.LOGS/'events-grouped.jsonl';path.write_text(''.join(json.dumps(x)+'\n' for x in rows))
|
|
with m.connect() as db:m.import_logs(db)
|
|
result=m.events({})
|
|
self.assertEqual(result['raw_total'],4)
|
|
self.assertEqual(result['total'],2)
|
|
first=next(x for x in result['items'] if x['component_id']=='gpu-a')
|
|
self.assertEqual(first['record_count'],3)
|
|
self.assertEqual(first['signal_reports'],2)
|
|
self.assertEqual(m.events({'view':['raw']})['total'],4)
|
|
original_id=first['id']
|
|
with path.open('a') as f:f.write(json.dumps(row('0000:03:00.0','PCIe Bus Error: severity=Correctable',now-10,990000000))+'\n')
|
|
with m.connect() as db:m.import_logs(db)
|
|
new=next(x for x in m.events({})['items'] if x['component_id']=='gpu-a')
|
|
self.assertEqual(new['id'],original_id)
|
|
self.assertEqual(new['signal_reports'],3)
|
|
|
|
def test_five_minute_quiet_gap_starts_another_incident(self):
|
|
now=int(time.time())
|
|
rows=[{'observed_at':m.iso(now),'first_at':m.iso(now-age),'last_at':m.iso(now-age),
|
|
'source':'kernel','boot_id':'oldboot','first_monotonic_us':str(1000000+i),
|
|
'last_monotonic_us':str(1000000+i),'device':'0000:03:00.0',
|
|
'category':'pcie_correctable','severity':'WARN','count':1,
|
|
'detail':'PCIe Bus Error: severity=Correctable'} for i,age in enumerate([1000,10])]
|
|
(m.LOGS/'events-gaps.jsonl').write_text(''.join(json.dumps(x)+'\n' for x in rows))
|
|
with m.connect() as db:m.import_logs(db)
|
|
self.assertEqual(m.events({})['total'],2)
|
|
|
|
def test_current_retry_is_warning_and_has_plain_action(self):
|
|
now=int(time.time())
|
|
m.LATEST.update({'gpus':[{'uuid':'gpu-a'},{'uuid':'gpu-b'}],'expected_gpu_count':2,
|
|
'issues':[{'category':'pcie_correctable','device':'0000:00:03.0'}],
|
|
'device_map':{'0000:00:03.0':{'id':'gpu-a','label':'第 1 张 P100','pci':'0000:03:00.0'}}})
|
|
row={'observed_at':m.iso(now),'first_at':m.iso(now-10),'last_at':m.iso(now-10),
|
|
'source':'kernel','boot_id':'test-current','first_monotonic_us':'990000000',
|
|
'last_monotonic_us':'990000000','device':'0000:00:03.0',
|
|
'category':'pcie_correctable','severity':'WARN','count':1,
|
|
'detail':'PCIe Bus Error: severity=Correctable'}
|
|
(m.LOGS/'events-active.jsonl').write_text(json.dumps(row)+'\n')
|
|
with m.connect() as db:m.import_logs(db)
|
|
item=m.events({})['items'][0]
|
|
self.assertEqual(item['severity'],'WARN')
|
|
self.assertEqual(item['state'],'active')
|
|
self.assertIn('显卡可用',item['state_label'])
|
|
self.assertIn('不能证明',item['meaning'])
|
|
self.assertEqual(len(item['next_steps']),3)
|
|
|
|
def test_problem_filter_hides_routine_information(self):
|
|
now=int(time.time())
|
|
rows=[{'observed_at':m.iso(now),'category':cat,'severity':severity,'detail':cat}
|
|
for cat,severity in [('boot','INFO'),('power','ERROR')]]
|
|
(m.LOGS/'events-levels.jsonl').write_text(''.join(json.dumps(x)+'\n' for x in rows))
|
|
with m.connect() as db:m.import_logs(db)
|
|
self.assertEqual(m.events({'severity':['problem']})['total'],1)
|
|
self.assertEqual(m.events({'severity':['INFO']})['total'],1)
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|