Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Analyzer/AnalyzerTool.cs
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public int Analyze(AnalyzeOptions options)
foundParser = true;
try
{
parser.Parse(file);
parser.Parse(file, displayRoot);
ReportProgress(relativePath, i, files.Count);
countSuccess++;
}
Expand Down
2 changes: 1 addition & 1 deletion Analyzer/Resources/Init.sql
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ WHERE m.type = 'Material';

INSERT INTO types (id, name) VALUES (-1, 'Scene');

PRAGMA user_version = 7;
PRAGMA user_version = 8;

PRAGMA synchronous = OFF;
PRAGMA journal_mode = MEMORY;
5 changes: 4 additions & 1 deletion Analyzer/SQLite/Handlers/ISQLiteHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,10 @@ public interface ISQLiteFileParser : IDisposable
{
void Init(SqliteConnection db);
bool CanParse(string filename);
void Parse(string filename);

// rootDirectory is the scanned input path the file was found under; names recorded in the
// database are relative to it, so same-named files in different sub-folders stay distinct.
void Parse(string filename, string rootDirectory);

// Called once after all files have been parsed, so a parser can write data that can only be
// determined from the complete set (e.g. dangling references). No-op for parsers that don't
Expand Down
2 changes: 1 addition & 1 deletion Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public bool CanParse(string filename)
return false;
}

public void Parse(string filename)
public void Parse(string filename, string rootDirectory)
{
// only init our writer if we are actually parsing a file
m_Writer.Init();
Expand Down
2 changes: 1 addition & 1 deletion Analyzer/SQLite/Parsers/ContentLayoutParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public bool CanParse(string filename)
return IsContentLayoutFile(filename);
}

public void Parse(string filename)
public void Parse(string filename, string rootDirectory)
{
ContentLayout layout;
using (var reader = File.OpenText(filename))
Expand Down
17 changes: 12 additions & 5 deletions Analyzer/SQLite/Parsers/SerializedFileParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,11 @@ public void Init(SqliteConnection db)
m_SerializedFileIdProvider, m_ContentFileDependencies);
}

public void Parse(string filename)
public void Parse(string filename, string rootDirectory)
{
// only init our writer if we are actually parsing a file
m_Writer.Init();
ProcessFile(filename, Path.GetDirectoryName(filename));
ProcessFile(filename, rootDirectory);
}

bool ShouldIgnoreFile(string file)
Expand Down Expand Up @@ -105,7 +105,12 @@ void ProcessFile(string file, string rootDirectory)

try
{
var archiveName = Path.GetRelativePath(rootDirectory, file);
// Naming an archive by its path relative to the scanned root keeps
// bundles that share a file name in different folders distinct (issue
// #149). Forward slashes make the value platform independent and match
// the name the AssetBundleManifest uses.
var archiveName = Path.GetRelativePath(rootDirectory, file)
.Replace(Path.DirectorySeparatorChar, '/');

m_Writer.BeginArchive(archiveName, new FileInfo(file).Length);

Expand Down Expand Up @@ -173,8 +178,10 @@ void ProcessFile(string file, string rootDirectory)
// This isn't a Unity Archive file, so process it as a SerializedFile.
// Note: The file has already been validated in CanParse() via SerializedFileDetector,
// so we're confident it's a valid SerializedFile at this point.
var relativePath = Path.GetRelativePath(rootDirectory, file);
m_Writer.WriteSerializedFile(relativePath, file, Path.GetDirectoryName(file));
//
// Unlike an archive, a SerializedFile is recorded under its bare file name: that is
// the name external references and serialized_files.id are keyed on (issue #149).
Comment thread
SkowronskiAndrew marked this conversation as resolved.
Outdated
m_Writer.WriteSerializedFile(Path.GetFileName(file), file, Path.GetDirectoryName(file));
}
}
}
Expand Down
9 changes: 8 additions & 1 deletion Documentation/analyzer-schema.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,9 +147,15 @@ is not inside an archive has no row here.
| Column | Type | Description |
|---|---|---|
| `id` | INTEGER | Analyzer-assigned id. Primary key. |
| `name` | TEXT | The archive's name on the file system. UNIQUE. |
| `name` | TEXT | The archive's path relative to the analyzed directory, with `/` separators. UNIQUE. |
| `file_size` | INTEGER | Size of the archive file in bytes. |

`name` is the path of the archive file relative to the directory that was analyzed, for example
`dlc/weapons/main`. `BuildPipeline.BuildAssetBundles` writes a bundle whose name is a path into a
matching folder structure, so the relative path is both unique and the name the bundle has in the
`AssetBundleManifest`. An archive passed directly on the command line, rather than found by scanning
a directory, is recorded under its bare file name.

`name` is UNIQUE and case-sensitive because analyze supports a single build at a time: two archives
with the same name would make every query ambiguous. A duplicate is detected while writing and
reported as an error; the constraint is the durable backstop. See
Expand Down Expand Up @@ -545,6 +551,7 @@ Any schema change - a new or changed table, view or column - must bump the pragm
| 5 | Added the `dangling_refs` table and view ([#85](https://github.com/Unity-Technologies/UnityDataTools/issues/85)) |
| 6 | `archives.name` is UNIQUE ([#51](https://github.com/Unity-Technologies/UnityDataTools/issues/51)) |
| 7 | Unity 6.6 `build_reports` columns and `build_report_content_*` tables ([#107](https://github.com/Unity-Technologies/UnityDataTools/issues/107)); `asset_name` / `asset_extension` columns on `build_report_source_assets` ([#110](https://github.com/Unity-Technologies/UnityDataTools/issues/110)) |
| 8 | `archives.name` is the path relative to the scanned directory, not the bare file name ([#149](https://github.com/Unity-Technologies/UnityDataTools/issues/149)) |

## Related documentation

Expand Down
13 changes: 13 additions & 0 deletions Documentation/command-analyze.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,15 @@ UnityDataTool analyze /path/to/bundles --skip-references --skip-crc

See also [Analyze Examples](analyze-examples.md).

## How archives are named

The `archives.name` column, and the `archive` column of the views built on it, holds the path of
the archive relative to the directory that was analyzed, using `/` separators. An AssetBundle name
may itself be a path, and `BuildPipeline.BuildAssetBundles` writes such a bundle into a matching
folder structure, so a build containing `dlc/weapons/main` and `dlc/armor/main` records those two
names rather than two bundles called `main`. An archive named directly on the command line, rather
than found by scanning a directory, is recorded under its bare file name.

Comment thread
SkowronskiAndrew marked this conversation as resolved.
Outdated
---

## What Can Be Analyzed
Expand Down Expand Up @@ -202,6 +211,10 @@ When analyze encounters a second file or archive with a name it has already proc
of the messages above, **skips that file or archive** (counting it as a failed file), and continues
with the rest of the input. The already-analyzed copy is kept; the duplicate's content is ignored.

Two bundles that share a file name in different sub-folders of the analyzed directory are not
duplicates: archives are named by their relative path, so `dlc/weapons/main` and `dlc/armor/main`
are both analyzed. See [How archives are named](#how-archives-are-named).

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good.

I think this example error above should be removed:

Skipping build2\assetbundle: Duplicate archive name 'assetbundle'. Each analyzed archive must have a unique name; only a single build can be analyzed at a time.

And

When analyze encounters a second file or archive with a name it has already processed, it prints one
of the messages above

It seems this section needs more revision to make sure it is accurate (duplicate archives are ok, but serialized files need to be unique. Hence it is still true that we cannot analyze two versions of the same build and AssetBundle variants trigger an error.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AGENT: Reworked the section. Renamed it to "Duplicate SerializedFile name", led with SerializedFile uniqueness as the real constraint, and dropped the claim that archive names must be unique.

I kept the Duplicate archive name example, but moved it into a paragraph that says when it now happens: archive names no longer have to be unique across folders, but two builds analyzed together still collide, and when each build is passed as its own input path the archive names collide too (each path is its own root, so both bundles are still named assetbundle). That case is still reachable and still covered by a test, so removing the message entirely would leave an error text that appears in output but nowhere in the docs. Happy to drop it if you'd rather the section only covered SerializedFiles.

This is expected when the input contains more than one build, and in these common cases:

| Cause | What to do |
Expand Down
55 changes: 55 additions & 0 deletions UnityDataTool.Tests/AnalyzeDuplicateNameTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ namespace UnityDataTools.UnityDataTool.Tests;
// single-line message instead of a raw "UNIQUE constraint failed" SQLite error. Covers the three
// scenarios from the issue: loose files, archives with the same name, and differently-named
// archives (hashed bundle names) that share the same inner SerializedFile.
//
// Also covers the opposite case (issue #149): bundles sharing a file name in different sub-folders
// of one scanned directory are distinct archives, not duplicates.
public class AnalyzeDuplicateNameTests
{
private string m_TestOutputFolder;
Expand Down Expand Up @@ -159,4 +162,56 @@ public async Task Analyze_DifferentArchiveNamesSharingSerializedFile_SkippedWith
WHERE name LIKE 'CAB-%' AND id IN (SELECT serialized_file FROM objects)",
1, "the shared inner SerializedFile should be analyzed only once");
}

// Issue #149: an AssetBundle name can be a path, so a build commonly contains several bundles
// with the same file name in different folders. They are recorded under their path relative to
// the scanned directory and are not treated as duplicates.
[Test]
public async Task Analyze_SameFileNameInDifferentFolders_NamedByRelativePath()
{
var sourceFolder = Path.Combine(m_AssetBundlesFolder, "2019.4.0f1");
var folderA = Path.Combine(m_TestOutputFolder, "dlc", "weapons");
var folderB = Path.Combine(m_TestOutputFolder, "dlc", "armor");
Directory.CreateDirectory(folderA);
Directory.CreateDirectory(folderB);
File.Copy(Path.Combine(sourceFolder, "assetbundle"), Path.Combine(folderA, "main"));
File.Copy(Path.Combine(sourceFolder, "scenes"), Path.Combine(folderB, "main"));
var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder);

var (exitCode, stderr) = await RunAnalyze(m_TestOutputFolder, "-o", databasePath);

Assert.AreEqual(0, exitCode);
StringAssert.DoesNotContain("Duplicate archive name", stderr);
StringAssert.DoesNotContain("UNIQUE constraint", stderr);

using var db = SQLTestHelper.OpenDatabase(databasePath);
SQLTestHelper.AssertQueryInt(db,
"SELECT COUNT(*) FROM archives WHERE name IN ('dlc/weapons/main', 'dlc/armor/main')",
2, "both bundles should be recorded under their relative path");
Assert.Greater(SQLTestHelper.QueryInt(db,
"SELECT COUNT(*) FROM object_view WHERE archive = 'dlc/weapons/main'"), 0);
Assert.Greater(SQLTestHelper.QueryInt(db,
"SELECT COUNT(*) FROM object_view WHERE archive = 'dlc/armor/main'"), 0);
}

// A file named directly on the command line has no scanned directory to be relative to, so it
// keeps its bare file name.
[Test]
public async Task Analyze_FileNamedDirectly_KeepsBareFileName()
{
var source = Path.Combine(m_AssetBundlesFolder, "2019.4.0f1", "assetbundle");
var nested = Path.Combine(m_TestOutputFolder, "dlc", "weapons");
Directory.CreateDirectory(nested);
var bundle = Path.Combine(nested, "main");
File.Copy(source, bundle);
var databasePath = SQLTestHelper.GetDatabasePath(m_TestOutputFolder);

var (exitCode, _) = await RunAnalyze(bundle, "-o", databasePath);

Assert.AreEqual(0, exitCode);

using var db = SQLTestHelper.OpenDatabase(databasePath);
SQLTestHelper.AssertQueryString(db, "SELECT name FROM archives", "main",
"a directly-named archive keeps its bare file name");
}
}
Loading