mirror of
https://github.com/storytold/craft-common-actions.git
synced 2026-11-20 23:41:13 +00:00
bf191401c5
- rust-cache: a saving leg's crates entry is <job>-crates-<os>-<arch>-<parent run>-0, from .ci-cache-meta/entry, so of two runs that restored the same entry only the first to finish saves (GitHub refuses an existing key). Nothing restored gives -0-0; a cold build writes -<run>-<attempt>. A saving leg gets CRAFT_CACHE_UNITS, a file for the build's Cargo JSON messages; the name is outside the prefixes rust-cache hashes into its key. - cache-trim: deletes every unit directory (<profile>/build/<package>/<hash>/) the record does not name, before rust-cache's post step saves; rust-cache's own cleanup matches package names only and keeps older builds of updated dependencies. No record, no change. - cache-prune: for a caller job with actions: write. Keeps the newest rolling crates entry per job/OS/arch, and of other rust-cache entries and named <name>-<sha256> entries removes an older one once nothing used it since the newest was created. watch: prunes as the run's saving jobs finish and ends when none is queued or running.
220 lines
10 KiB
Python
220 lines
10 KiB
Python
"""actions/cache-prune: one live entry per family, deleted as soon as its successor is saved."""
|
|
from pathlib import Path
|
|
import contextlib
|
|
import io
|
|
import re
|
|
import sys
|
|
import unittest
|
|
import unittest.mock
|
|
|
|
import yaml
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'actions/cache-prune'))
|
|
import prune # noqa: E402
|
|
|
|
PROFILE = '7' * 64
|
|
OTHER_PROFILE = '8' * 64
|
|
|
|
|
|
def entry(id_, key, created, accessed=None, size=10**9):
|
|
return {'id': id_, 'key': key, 'created_at': f'2026-11-19T{created}:00Z',
|
|
'last_accessed_at': f'2026-11-19T{accessed or created}:00Z', 'size_in_bytes': size}
|
|
|
|
|
|
def crates(run, attempt='1', profile=PROFILE):
|
|
return f'v2-rust-{profile}-gate-crates-Linux-x64-{run}-{attempt}-Linux-x64-d843ffbc-3e8a8849'
|
|
|
|
|
|
class FamilyTests(unittest.TestCase):
|
|
def test_families(self):
|
|
self.assertEqual(prune.family(crates('44600646541'), []), ('rolling', 'v2-rust-gate-crates-Linux-x64'))
|
|
self.assertEqual(prune.family(crates('812', '0'), []), ('rolling', 'v2-rust-gate-crates-Linux-x64'))
|
|
self.assertEqual(prune.family(f'v2-rust-{PROFILE}-gate-Windows_NT-x64-ae62fac7-79024277', []),
|
|
('replaced', 'v2-rust-gate-Windows_NT-x64'))
|
|
self.assertEqual(prune.family('v1-rust-ci-web-web-Linux-x64-304bc087-3e8a8849', []),
|
|
('replaced', 'v1-rust-ci-web-web-Linux-x64'))
|
|
self.assertEqual(prune.family('corpus-' + 'c' * 64, ['corpus']), ('replaced', 'corpus'))
|
|
|
|
def test_other_entries_are_never_touched(self):
|
|
for key in ('corpus-' + 'c' * 64, # content keys only when named
|
|
'apt-debs-v1-ubuntu24-' + '2' * 64 + '-20261116.376.1',
|
|
'harden-runner-cacheKey',
|
|
'determinatesystem-nix-installer-x86_64-linux-88d8c6a2d0f6cbacb0d4301ab84ecba4',
|
|
'sccache-' + 'a' * 64):
|
|
with self.subTest(key=key):
|
|
self.assertIsNone(prune.family(key, []))
|
|
|
|
|
|
class PlanTests(unittest.TestCase):
|
|
def deleted(self, entries, names=()):
|
|
return sorted(e['id'] for e, _ in prune.plan(entries, list(names)))
|
|
|
|
def test_only_the_newest_rolling_entry_stays(self):
|
|
entries = [entry(1, crates('100'), '10:00'), entry(2, crates('100', '0'), '11:00', '11:30'),
|
|
entry(3, crates('101', '0'), '12:00'), entry(4, crates('99', profile=OTHER_PROFILE), '09:00')]
|
|
self.assertEqual(self.deleted(entries), [1, 2, 4])
|
|
|
|
def test_a_replaced_entry_goes_once_nothing_used_it_since(self):
|
|
old_web = entry(1, 'v1-rust-ci-web-web-Linux-x64-304bc087-aaaaaaaa', '10:00', '10:05')
|
|
new_web = entry(2, 'v1-rust-ci-web-web-Linux-x64-304bc087-bbbbbbbb', '11:00')
|
|
old_corpus = entry(3, 'corpus-' + 'a' * 64, '09:00', '11:10') # still restored after the new one
|
|
new_corpus = entry(4, 'corpus-' + 'b' * 64, '11:00')
|
|
self.assertEqual(self.deleted([old_web, new_web, old_corpus, new_corpus], ['corpus']), [1])
|
|
|
|
def test_os_and_job_are_separate_families(self):
|
|
entries = [entry(1, f'v2-rust-{PROFILE}-gate-Windows_NT-x64-ae62fac7-79024277', '10:00'),
|
|
entry(2, 'v1-rust-ci-web-web-Linux-x64-304bc087-3e8a8849', '11:00'),
|
|
entry(3, crates('100').replace('Linux-x64', 'Windows_NT-x64'), '09:00'),
|
|
entry(4, crates('101'), '12:00')]
|
|
self.assertEqual(self.deleted(entries), [])
|
|
|
|
|
|
class FakeApi:
|
|
def __init__(self, entries, job_polls, late=(), failures=0):
|
|
self.entries = list(entries)
|
|
self.job_polls = list(job_polls)
|
|
self.late = list(late) # (listing from which it shows, entry): a cache list that lags
|
|
self.failures = failures
|
|
self.deleted = []
|
|
self.listed = 0
|
|
|
|
def caches(self, ref):
|
|
self.listed += 1
|
|
if self.failures:
|
|
self.failures -= 1
|
|
raise prune.urllib.error.HTTPError('u', 502, 'Bad Gateway', None, None)
|
|
shown = self.entries + [e for n, e in self.late if self.listed >= n]
|
|
return [e for e in shown if e['id'] not in self.deleted]
|
|
|
|
def delete(self, cache_id):
|
|
self.deleted.append(cache_id)
|
|
|
|
def jobs(self, run, attempt):
|
|
return self.job_polls.pop(0)
|
|
|
|
|
|
def job(id_, name, status, runner='r', started=None, conclusion=None):
|
|
found = {'id': id_, 'name': name, 'status': status, 'runner_name': runner}
|
|
if started:
|
|
found.update(started_at=f'2026-11-19T{started}:00Z', conclusion=conclusion or 'success')
|
|
return found
|
|
|
|
|
|
SAVING = [re.compile(p) for p in (r'^gate( / build \(.*\))?$', r'^windows( / build \(.*\))?$')]
|
|
|
|
|
|
class WatchTests(unittest.TestCase):
|
|
def watch(self, api, patterns=SAVING, dry_run=False, minutes=110, clock=None):
|
|
self.sleeps = 0
|
|
|
|
def sleep(_):
|
|
self.sleeps += 1
|
|
with contextlib.redirect_stdout(io.StringIO()) as out:
|
|
prune.watch(api, 'refs/heads/main', [], dry_run, '1', '1', patterns, 'me', minutes,
|
|
sleep=sleep, clock=clock or (lambda: 0))
|
|
return out.getvalue()
|
|
|
|
def test_prunes_when_a_saving_job_finishes_and_ends_when_none_is_left(self):
|
|
me = job(0, 'cache prune', 'in_progress', runner='me')
|
|
polls = [
|
|
[me, job(1, 'select platforms', 'in_progress')],
|
|
[me, job(1, 'select platforms', 'completed'), job(2, 'gate / build (ubuntu-24.04)', 'in_progress')],
|
|
[me, job(2, 'gate / build (ubuntu-24.04)', 'completed'), job(3, 'windows', 'completed'),
|
|
job(4, 'gate / test 1/4 (ubuntu-24.04)', 'in_progress')],
|
|
]
|
|
api = FakeApi([entry(1, crates('100'), '10:00'), entry(2, crates('100', '0'), '11:00', size=4 * 10**9)], polls)
|
|
out = self.watch(api)
|
|
self.assertEqual(api.deleted, [1])
|
|
self.assertEqual(api.job_polls, []) # it ended with a shard still running
|
|
self.assertEqual(api.listed, 2) # the first pass, and the one after the build
|
|
self.assertIn(f"delete {crates('100')} (1.00 GB)", out)
|
|
|
|
def test_a_saving_job_not_listed_yet_keeps_it_going(self):
|
|
polls = [[job(1, 'gate / build (ubuntu-24.04)', 'completed')],
|
|
[job(1, 'gate / build (ubuntu-24.04)', 'completed'), job(2, 'windows / build (windows-latest)', 'in_progress')],
|
|
[job(1, 'gate / build (ubuntu-24.04)', 'completed'), job(2, 'windows / build (windows-latest)', 'completed')]]
|
|
api = FakeApi([], polls)
|
|
self.watch(api)
|
|
self.assertEqual(api.job_polls, [])
|
|
|
|
def test_without_patterns_it_ends_after_two_idle_polls(self):
|
|
polls = [[job(1, 'a', 'in_progress')], [job(1, 'a', 'completed')], [job(1, 'a', 'completed')], [job(9, 'never read', 'queued')]]
|
|
api = FakeApi([], polls)
|
|
self.watch(api, patterns=[])
|
|
self.assertEqual(len(api.job_polls), 1)
|
|
|
|
def test_the_last_pass_waits_for_the_last_saving_jobs_entry(self):
|
|
# The cache list shows the Windows entry two listings after the job completed: the old one
|
|
# must still go in this run.
|
|
polls = [[job(1, 'gate / build (x)', 'completed', started='10:00'),
|
|
job(2, 'windows / build (w)', 'completed', started='10:30')]]
|
|
old = entry(1, f'v2-rust-{PROFILE}-gate-Windows_NT-x64-ae62fac7-79024277', '09:00')
|
|
new = entry(2, f'v2-rust-{PROFILE}-gate-Windows_NT-x64-ae62fac7-12345678', '11:00')
|
|
api = FakeApi([old], polls, late=[(4, new)])
|
|
self.watch(api)
|
|
self.assertEqual(api.deleted, [1])
|
|
self.assertEqual(api.listed, 4)
|
|
|
|
def test_the_last_pass_gives_up_when_no_entry_appears(self):
|
|
# A job whose key already existed saves nothing.
|
|
polls = [[job(1, 'gate / build (x)', 'completed', started='10:00'), job(2, 'windows', 'completed', started='10:00')]]
|
|
api = FakeApi([entry(1, crates('100'), '09:00')], polls)
|
|
self.watch(api)
|
|
self.assertEqual(api.listed, 2 + prune.FINAL_POLLS)
|
|
|
|
def test_a_skipped_saving_job_is_not_waited_for(self):
|
|
polls = [[job(1, 'gate / build (x)', 'completed', started='10:00', conclusion='skipped'),
|
|
job(2, 'windows', 'completed', started='10:00', conclusion='skipped')]]
|
|
api = FakeApi([], polls)
|
|
self.watch(api)
|
|
self.assertEqual(api.listed, 2)
|
|
|
|
def test_a_failed_pass_warns_and_the_next_one_prunes(self):
|
|
polls = [[job(1, 'gate / build (x)', 'in_progress')],
|
|
[job(1, 'gate / build (x)', 'completed'), job(2, 'windows', 'completed')]]
|
|
api = FakeApi([entry(1, crates('100'), '10:00'), entry(2, crates('101'), '11:00')], polls, failures=1)
|
|
out = self.watch(api)
|
|
self.assertIn('::warning::cache-prune: a pass failed', out)
|
|
self.assertEqual(api.deleted, [1])
|
|
|
|
def test_it_ends_with_a_pass_at_the_deadline(self):
|
|
ticks = iter(range(0, 10**6, 600)) # ten minutes per poll
|
|
polls = [[job(1, 'gate / build (x)', 'in_progress')]] * 3
|
|
api = FakeApi([], polls)
|
|
out = self.watch(api, minutes=25, clock=lambda: next(ticks))
|
|
self.assertIn('still watching after 25 minutes', out)
|
|
self.assertEqual(api.listed, 2)
|
|
|
|
def test_a_failure_that_stops_the_script_is_a_warning(self):
|
|
env = {'PRUNE_REPO': 'o/r', 'GH_TOKEN': 't', 'PRUNE_REF': 'refs/heads/main'}
|
|
|
|
def broken(*_):
|
|
raise RuntimeError('boom')
|
|
original, prune.prune = prune.prune, broken
|
|
self.addCleanup(setattr, prune, 'prune', original)
|
|
with unittest.mock.patch.dict(prune.os.environ, env), contextlib.redirect_stdout(io.StringIO()) as out:
|
|
self.assertEqual(prune.main(), 1)
|
|
self.assertIn("::warning::cache-prune stopped: RuntimeError('boom')", out.getvalue())
|
|
|
|
def test_dry_run_deletes_nothing(self):
|
|
api = FakeApi([entry(1, crates('100'), '10:00'), entry(2, crates('101'), '11:00')],
|
|
[[job(1, 'gate / build (x)', 'completed'), job(2, 'windows', 'completed')]])
|
|
out = self.watch(api, dry_run=True)
|
|
self.assertEqual(api.deleted, [])
|
|
self.assertIn('would delete', out)
|
|
|
|
|
|
class DefinitionTests(unittest.TestCase):
|
|
def test_the_action_runs_the_script_with_its_inputs(self):
|
|
action = yaml.load((ROOT / 'actions/cache-prune/action.yml').read_text(), Loader=yaml.BaseLoader)
|
|
env = action['runs']['steps'][0]['env']
|
|
self.assertEqual(env['GH_TOKEN'], '${{ inputs.token }}')
|
|
self.assertEqual(action['inputs']['watch']['default'], 'false')
|
|
self.assertEqual(action['inputs']['dry-run']['default'], 'false')
|
|
self.assertNotIn('${{', action['runs']['steps'][0]['run'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|