Skip to content

Commit f9e95e3

Browse files
DeusDataDeusData
authored
Merge pull request #1688 from DeusData/fix/vt-gate-honors-withheld
fix(ci): teach the VirusTotal gate the withheld-executables manifest
2 parents c1a5de3 + 8eff872 commit f9e95e3

2 files changed

Lines changed: 90 additions & 11 deletions

File tree

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,7 @@ jobs:
350350
VT_EXPECTED_SCAN_SET: binaries/scan-set.tsv
351351
VT_ASSOCIATIONS: binaries/associations.tsv
352352
VT_RESULTS_PATH: binaries/vt-results.tsv
353+
VT_WITHHELD: binaries/virustotal-withheld.tsv
353354
MIN_ENGINES: 50
354355
run: scripts/ci/check-virustotal.sh
355356

@@ -362,6 +363,7 @@ jobs:
362363
binaries/associations.tsv
363364
binaries/scan-set.tsv
364365
binaries/vt-results.tsv
366+
binaries/virustotal-withheld.tsv
365367
if-no-files-found: warn
366368

367369
- name: Publish durable public VirusTotal evidence

‎scripts/ci/check-virustotal.sh‎

Lines changed: 88 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ class ExpectedObject:
106106
size: int
107107
association_count: int
108108
association_kinds: frozenset[str]
109+
withheld: bool
109110
110111
@property
111112
def requires_microsoft(self) -> bool:
@@ -188,7 +189,54 @@ def parse_versioned_tsv(
188189
return metadata, rows
189190
190191
191-
def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]:
192+
def load_withheld(raw_path: str) -> Dict[str, str]:
193+
"""Objects deliberately removed from the surface scan (VT_WITHHELD).
194+
195+
exclude-rescanned-selected-objects.sh deletes the SELECTED executables from
196+
the objects directory before the upload — their bytes were already scanned
197+
as release candidates, identity is settled by sha256, and re-submitting
198+
identical bytes re-rolls a probabilistic classifier (measured on v0.10.5:
199+
verdicts flipped in both directions within the hour). The gate accepts a
200+
withheld object only when this manifest vouches for it BY HASH; everything
201+
else keeps the strict on-disk contract. An absent manifest keeps the
202+
original behavior everywhere else this gate runs (candidate stage,
203+
dry-run), where nothing is ever withheld."""
204+
path = pathlib.Path(raw_path).absolute()
205+
if path.is_symlink() or not path.is_file():
206+
raise GateError(f"withheld manifest is not a regular file: {path}")
207+
if path.stat().st_size > 1024 * 1024:
208+
raise GateError(f"withheld manifest is unexpectedly large: {path}")
209+
lines = path.read_text(encoding="utf-8").splitlines()
210+
if not lines or lines[0] != "# cbm-virustotal-withheld-v1":
211+
raise GateError("withheld manifest marker is missing")
212+
cursor = 1
213+
metadata: Dict[str, str] = {}
214+
while cursor < len(lines) and lines[cursor].startswith("# "):
215+
key, separator, value = lines[cursor][2:].partition("=")
216+
if not separator or not key or key in metadata or not value:
217+
raise GateError(f"malformed withheld metadata: {lines[cursor]}")
218+
metadata[key] = value
219+
cursor += 1
220+
if metadata.get("reason") != "already-scanned-as-candidate":
221+
raise GateError("withheld manifest does not state the accepted reason")
222+
if cursor >= len(lines) or lines[cursor] != "sha256\tobject":
223+
raise GateError("withheld manifest header is malformed")
224+
withheld: Dict[str, str] = {}
225+
for line in lines[cursor + 1 :]:
226+
sha256, separator, name = line.partition("\t")
227+
if not separator or SHA256_RE.fullmatch(sha256) is None or not name or "/" in name:
228+
raise GateError(f"malformed withheld row: {line}")
229+
if sha256 in withheld:
230+
raise GateError(f"duplicate withheld sha256: {sha256}")
231+
withheld[sha256] = name
232+
if not withheld:
233+
raise GateError("withheld manifest names no objects")
234+
return withheld
235+
236+
237+
def load_expected(
238+
path: pathlib.Path, withheld: Dict[str, str]
239+
) -> Tuple[List[ExpectedObject], int]:
192240
metadata, rows = parse_versioned_tsv(
193241
path,
194242
marker="cbm-release-scan-set-v2",
@@ -230,15 +278,26 @@ def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]:
230278
):
231279
raise GateError(f"unassociated object in expected set: {scan_path}")
232280
local_path = path.parent.joinpath(*pure.parts)
233-
try:
234-
mode = local_path.lstat().st_mode
235-
except FileNotFoundError as error:
236-
raise GateError(f"expected scan object is missing: {scan_path}") from error
237-
if not stat.S_ISREG(mode):
238-
raise GateError(f"expected scan object is not a regular file: {scan_path}")
239-
actual_size = local_path.stat().st_size
240-
if actual_size != size or sha256_file(local_path) != sha256:
241-
raise GateError(f"expected scan object changed after extraction: {scan_path}")
281+
is_withheld = sha256 in withheld
282+
if is_withheld:
283+
if withheld[sha256] != pure.parts[1]:
284+
raise GateError(
285+
f"withheld manifest names a different object for this hash: {scan_path}"
286+
)
287+
if os.path.lexists(local_path):
288+
raise GateError(
289+
f"withheld object is still present in the scan directory: {scan_path}"
290+
)
291+
else:
292+
try:
293+
mode = local_path.lstat().st_mode
294+
except FileNotFoundError as error:
295+
raise GateError(f"expected scan object is missing: {scan_path}") from error
296+
if not stat.S_ISREG(mode):
297+
raise GateError(f"expected scan object is not a regular file: {scan_path}")
298+
actual_size = local_path.stat().st_size
299+
if actual_size != size or sha256_file(local_path) != sha256:
300+
raise GateError(f"expected scan object changed after extraction: {scan_path}")
242301
association_total += association_count
243302
objects.append(
244303
ExpectedObject(
@@ -248,10 +307,17 @@ def load_expected(path: pathlib.Path) -> Tuple[List[ExpectedObject], int]:
248307
size=size,
249308
association_count=association_count,
250309
association_kinds=frozenset(kinds),
310+
withheld=is_withheld,
251311
)
252312
)
253313
if association_total != associations:
254314
raise GateError("scan-set association counts do not match manifest metadata")
315+
expected_hashes = {item.sha256 for item in objects}
316+
spurious = sorted(set(withheld) - expected_hashes)
317+
if spurious:
318+
raise GateError(f"withheld manifest names hashes outside the expected scan set: {spurious}")
319+
if all(item.withheld for item in objects):
320+
raise GateError("every expected object is withheld; the surface scan would cover nothing")
255321
return objects, associations
256322
257323
@@ -363,6 +429,7 @@ def parse_action_output(
363429
objects: Sequence[ExpectedObject],
364430
manifest: pathlib.Path,
365431
) -> List[Submission]:
432+
objects = [item for item in objects if not item.withheld]
366433
if not raw:
367434
raise GateError("VirusTotal action output is empty")
368435
aliases = output_aliases(objects, manifest)
@@ -697,7 +764,10 @@ def main() -> None:
697764
raise GateError(f"unsafe pre-existing VT results path: {results_path}")
698765
results_path.unlink()
699766
700-
expected, associations = load_expected(expected_manifest)
767+
withheld_path = os.environ.get("VT_WITHHELD", "")
768+
withheld = load_withheld(withheld_path) if withheld_path else {}
769+
770+
expected, associations = load_expected(expected_manifest, withheld)
701771
validate_associations(
702772
associations_manifest,
703773
objects=expected,
@@ -708,6 +778,13 @@ def main() -> None:
708778
objects=expected,
709779
manifest=expected_manifest,
710780
)
781+
withheld_count = sum(1 for item in expected if item.withheld)
782+
if withheld_count:
783+
print(
784+
f"=== {withheld_count} expected object(s) withheld from the surface scan "
785+
f"(already scanned as candidates; identity settled by sha256, manifest: "
786+
f"{withheld_path}) ==="
787+
)
711788
print(
712789
f"=== VirusTotal exact-set gate: {len(submissions)} distinct objects, "
713790
f"{associations} associations, >= {min_engines} decisive engines each ==="

0 commit comments

Comments
 (0)