diff --git a/Library/DiscUtils.Udf/DiscUtils.Udf.csproj b/Library/DiscUtils.Udf/DiscUtils.Udf.csproj index 5ecdc7a16..832c9a0e9 100644 --- a/Library/DiscUtils.Udf/DiscUtils.Udf.csproj +++ b/Library/DiscUtils.Udf/DiscUtils.Udf.csproj @@ -1,6 +1,6 @@  - Managed .NET reader for UDF optical-disc filesystems and their files, directories and extended attributes. + Managed .NET reader and virtual UDF 2.01 image builder for optical-disc filesystems. DiscUtils;Filesystem;Optical;UDF diff --git a/Library/DiscUtils.Udf/README.md b/Library/DiscUtils.Udf/README.md index ff447fc96..6e64cfef7 100644 --- a/Library/DiscUtils.Udf/README.md +++ b/Library/DiscUtils.Udf/README.md @@ -1,6 +1,6 @@ # LTRData.DiscUtils.Udf -Managed .NET reader for UDF optical-disc filesystems and their files, directories and extended attributes. +Managed .NET reader and image builder for UDF optical-disc filesystems. Part of the [LTRData DiscUtils fork](https://github.com/LTRData/DiscUtils). Package IDs use `LTRData.DiscUtils`; C# namespaces remain `DiscUtils`. @@ -15,13 +15,19 @@ dotnet add package LTRData.DiscUtils.Udf | Capability | Support | | --- | --- | | Read files and directories | Yes | -| Create a filesystem | No | +| Create a filesystem | UDF 2.01 with UdfBuilder | | Modify an existing filesystem | No | ## Usage notes UdfReader exposes files and extended attributes. Size, UsedSpace and AvailableSpace are not implemented. For automatic ISO/UDF detection, reference LTRData.DiscUtils.OpticalDisk and register its providers; this package has no generated Formats entry point. +UdfBuilder creates a read-only UDF 2.01 image with one physical partition and 2048-byte logical blocks. Files are limited to 234 short allocation descriptors (approximately 234 GiB per file); the partition is limited to approximately 8 TiB. The builder preserves names, directories and file contents, but does not import source filesystem attributes, extended attributes or permissions. + +`Build()` and `BuildAsync(cancellationToken)` return a seekable virtual image as a `Stream`. Construction creates metadata in memory and opens no file payloads. Every image owns the streams returned by its source factories; each factory must return a new readable, seekable stream of the declared length on every call. Async image reads use the source stream's async methods. Separate builds have independent positions; a single image stream must not be read concurrently. + +The source-path overload checks length and last-write time whenever it opens the file. These checks do not make a content snapshot: keep source files and retained byte arrays unchanged for the image's lifetime. Unexpected truncation fails, and reads never include source data beyond a declared extent. The overloads that build to an output file or stream materialize the entire image, including payloads and zero padding. + ## Example ```csharp @@ -36,6 +42,20 @@ foreach (var file in fs.Root.GetFiles()) Console.WriteLine(file.FullName); ``` +Create a virtual image from existing files: + +```csharp +using System.IO; +using DiscUtils.Udf; + +var builder = new UdfBuilder { VolumeIdentifier = "DISC" }; +builder.AddDirectory("BDMV/EMPTY"); +builder.AddFile("BDMV/index.bdmv", Path.Combine("source", "BDMV", "index.bdmv")); +builder.AddFile("readme.txt", System.Text.Encoding.UTF8.GetBytes("Virtual UDF image")); +using Stream image = builder.Build(); +// Serve reads/seeks from image, or materialize it with builder.Build(output). +``` + ## Related packages - [LTRData.DiscUtils.OpticalDisk](https://www.nuget.org/packages/LTRData.DiscUtils.OpticalDisk) diff --git a/Library/DiscUtils.Udf/UdfBuilder.cs b/Library/DiscUtils.Udf/UdfBuilder.cs new file mode 100644 index 000000000..022acf47f --- /dev/null +++ b/Library/DiscUtils.Udf/UdfBuilder.cs @@ -0,0 +1,469 @@ +// SPDX-License-Identifier: MIT +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using DiscUtils.Streams; +using DiscUtils.Streams.Compatibility; + +namespace DiscUtils.Udf; + +/// +/// Builds a read-only UDF 2.01 image without copying file contents. +/// +/// +/// The image uses one physical partition and 2048-byte logical blocks. +/// Files are limited to 234 allocation descriptors (approximately 234 GiB). +/// Call to obtain a seekable virtual image; +/// the overloads taking an output file or stream materialize the entire image. +/// +public sealed class UdfBuilder : StreamBuilder +{ + private const int BlockSize = 2048; + private const uint PartitionStart = 291; + private const long MaximumExtentLength = (1L << 30) - BlockSize; + private const long MaximumFileLength = 234 * MaximumExtentLength; + private readonly Entry _root = new Entry(string.Empty, null, null, 0); + private string _volumeIdentifier = "DISC"; + private string _volumeSetIdentifier = Guid.NewGuid().ToString("N"); + + /// Gets or sets the volume identifier. + public string VolumeIdentifier + { + get { return _volumeIdentifier; } + set + { + UdfImageWriter.EncodeDString(value, 32); + _volumeIdentifier = value; + } + } + + /// Gets or sets a volume-set identifier, unique to this image and stable across reads. + /// The first sixteen characters must be hexadecimal digits, as required by UDF 2.01. + public string VolumeSetIdentifier + { + get { return _volumeSetIdentifier; } + set + { + UdfImageWriter.EncodeDString(value, 128); + if ( + value.Length < 16 + || value.Take(16).Any(character => !Uri.IsHexDigit(character)) + ) + { + throw new ArgumentException( + "A volume-set identifier must begin with sixteen hexadecimal digits.", + nameof(value) + ); + } + + _volumeSetIdentifier = value; + } + } + + /// Gets or sets the UTC timestamp used for the image metadata. + public DateTime RecordingTime { get; set; } = DateTime.UtcNow; + + /// Adds an empty directory, creating its parents as necessary. + /// The relative path inside the image. + public void AddDirectory(string name) + { + GetDirectory(SplitPath(name)); + } + + /// Adds a file whose contents are opened only when the image is read. + /// The relative path inside the image. + /// The existing file to read. + public void AddFile(string name, string sourcePath) + { + var file = new FileInfo(sourcePath); + long length = file.Length; + DateTime lastWrite = file.LastWriteTimeUtc; + AddFile( + name, + length, + () => + { + var current = new FileInfo(file.FullName); + if (current.Length != length || current.LastWriteTimeUtc != lastWrite) + { + throw new IOException( + "A source file has changed since the image was built." + ); + } + + return new FileStream( + file.FullName, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 4096, + useAsync: true + ); + } + ); + } + + /// Adds a byte array as a file. + /// The relative path inside the image. + /// The file contents, retained by reference. + public void AddFile(string name, byte[] content) + { + if (content == null) + { + throw new ArgumentNullException(nameof(content)); + } + + AddFile(name, content.LongLength, () => new MemoryStream(content, false)); + } + + /// Adds a lazily opened file with a known length. + /// The relative path inside the image. + /// The length of the source in bytes. + /// Creates a new readable, seekable stream on every call. + /// The image owns each returned stream and uses its asynchronous read methods for async image reads. + public void AddFile(string name, long length, Func openStream) + { + if (openStream == null) + { + throw new ArgumentNullException(nameof(openStream)); + } + + if (length < 0 || length > MaximumFileLength) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + string[] parts = SplitPath(name); + Entry parent = GetDirectory(parts.Take(parts.Length - 1)); + string leaf = parts[parts.Length - 1]; + if (parent.Children.ContainsKey(leaf)) + { + throw new IOException("An entry already exists at this path."); + } + + parent.Children.Add(leaf, new Entry(leaf, parent, openStream, length)); + } + + /// + public override Task BuildAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return base.BuildAsync(cancellationToken); + } + + /// + protected override List FixExtents(out long totalLength) + { + var entries = new List(); + Flatten(_root, entries); + uint nextBlock = 2; // File set descriptor and its terminator. + ulong nextId = 16; + foreach (Entry entry in entries) + { + entry.Icb = nextBlock++; + entry.UniqueId = entry == _root ? 0 : nextId++; + if (entry.IsDirectory) + { + entry.Length = + 40 + + entry.Children.Values.Sum(child => + (38 + UdfImageWriter.EncodeName(child.Name).Length + 3) / 4 * 4 + ); + } + } + + foreach ( + Entry entry in entries + .Where(entry => entry.IsDirectory) + .Concat(entries.Where(entry => !entry.IsDirectory)) + ) + { + entry.DataBlock = nextBlock; + nextBlock = checked(nextBlock + Blocks(entry.Length)); + } + + uint partitionLength = nextBlock; + uint lastBlock = checked(PartitionStart + partitionLength + 256); + totalLength = ((long)lastBlock + 1) * BlockSize; + var result = new List(); + var writer = new UdfImageWriter(_volumeIdentifier, _volumeSetIdentifier, RecordingTime); + + AddBytes(result, 16, UdfImageWriter.VolumeRecognition()); + AddBytes(result, 256, writer.Anchor(256)); + for (uint sequenceStart = 257; sequenceStart <= 273; sequenceStart += 16) + { + AddBytes(result, sequenceStart, writer.PrimaryVolume(sequenceStart)); + AddBytes(result, sequenceStart + 1, writer.ImplementationVolume(sequenceStart + 1)); + AddBytes( + result, + sequenceStart + 2, + writer.Partition(sequenceStart + 2, PartitionStart, partitionLength) + ); + AddBytes(result, sequenceStart + 3, writer.LogicalVolume(sequenceStart + 3)); + AddBytes(result, sequenceStart + 4, writer.UnallocatedSpace(sequenceStart + 4)); + AddBytes(result, sequenceStart + 5, UdfImageWriter.Terminator(sequenceStart + 5)); + } + + AddBytes( + result, + 289, + writer.Integrity( + partitionLength, + nextId, + (uint)entries.Count(entry => !entry.IsDirectory), + (uint)entries.Count(entry => entry.IsDirectory) + ) + ); + AddBytes(result, 290, UdfImageWriter.Terminator(290)); + AddBytes(result, PartitionStart, writer.FileSet(_root.Icb)); + AddBytes(result, PartitionStart + 1, UdfImageWriter.Terminator(1)); + + foreach (Entry entry in entries) + { + ushort links = entry.IsDirectory + ? checked((ushort)(1 + entry.Children.Values.Count(child => child.IsDirectory))) + : (ushort)1; + AddBytes( + result, + PartitionStart + entry.Icb, + writer.FileEntry( + entry.Icb, + entry.DataBlock, + entry.Length, + entry.IsDirectory, + links, + entry.UniqueId + ) + ); + if (entry.IsDirectory) + { + var bytes = new byte[checked((int)entry.Length)]; + int position = writer.FileIdentifier( + bytes, + 0, + entry.DataBlock, + string.Empty, + (entry.Parent ?? entry).Icb, + (entry.Parent ?? entry).UniqueId, + true, + true + ); + foreach (Entry child in entry.Children.Values) + { + position += writer.FileIdentifier( + bytes, + position, + entry.DataBlock + (uint)(position / BlockSize), + child.Name, + child.Icb, + child.UniqueId, + child.IsDirectory, + false + ); + } + + AddBytes(result, PartitionStart + entry.DataBlock, bytes); + } + else if (entry.Length > 0) + { + result.Add( + new SourceExtent( + ((long)PartitionStart + entry.DataBlock) * BlockSize, + entry.Length, + entry.OpenStream! + ) + ); + } + } + + AddBytes(result, lastBlock - 256, writer.Anchor(lastBlock - 256)); + AddBytes(result, lastBlock, writer.Anchor(lastBlock)); + return result; + } + + private static uint Blocks(long length) + { + return checked((uint)((length + BlockSize - 1) / BlockSize)); + } + + private static void AddBytes(List extents, uint block, byte[] bytes) + { + extents.Add(new BuilderBufferExtent((long)block * BlockSize, bytes)); + } + + private static string[] SplitPath(string name) + { + if (string.IsNullOrEmpty(name) || name[0] == '/' || name[0] == '\\') + { + throw new ArgumentException( + "An image path must be nonempty and relative.", + nameof(name) + ); + } + + string[] parts = name.Replace('\\', '/').Split('/'); + foreach (string part in parts) + { + if (part.Length == 0 || part == "." || part == ".." || part.IndexOf('\0') >= 0) + { + throw new ArgumentException("Invalid image path component.", nameof(name)); + } + + UdfImageWriter.EncodeName(part); + } + + return parts; + } + + private Entry GetDirectory(IEnumerable parts) + { + Entry parent = _root; + foreach (string part in parts) + { + if (!parent.Children.TryGetValue(part, out var child)) + { + child = new Entry(part, parent, null, 0); + parent.Children.Add(part, child); + } + + if (!child.IsDirectory) + { + throw new IOException("A file already exists at this path."); + } + + parent = child; + } + + return parent; + } + + private static void Flatten(Entry entry, List entries) + { + entries.Add(entry); + foreach (Entry child in entry.Children.Values) + { + Flatten(child, entries); + } + } + + private sealed class Entry + { + public Entry(string name, Entry? parent, Func? openStream, long length) + { + Name = name; + Parent = parent; + OpenStream = openStream; + Length = length; + } + + public string Name { get; } + public Entry? Parent { get; } + public Func? OpenStream { get; } + public bool IsDirectory + { + get { return OpenStream == null; } + } + public SortedDictionary Children { get; } = + new SortedDictionary(StringComparer.Ordinal); + public long Length { get; set; } + public uint Icb { get; set; } + public uint DataBlock { get; set; } + public ulong UniqueId { get; set; } + } + + private sealed class SourceExtent : BuilderExtent + { + private readonly Func _open; + private Stream? _stream; + + public SourceExtent(long start, long length, Func open) + : base(start, length) + { + _open = open; + } + + public override void PrepareForRead() + { + _stream = _open(); + try + { + if (_stream == null || !_stream.CanRead || !_stream.CanSeek || _stream.Length != Length) + { + throw new IOException( + "The source must be readable, seekable, and retain its declared length." + ); + } + } + catch + { + DisposeReadState(); + throw; + } + } + + public override int Read(long diskOffset, byte[] block, int offset, int count) + { + count = PrepareSourceRead(diskOffset, count); + return CheckRead(_stream!.Read(block, offset, count), count); + } + + public override int Read(long diskOffset, Span block) + { + int count = PrepareSourceRead(diskOffset, block.Length); + return CheckRead(_stream!.Read(block[..count]), count); + } + + public override ValueTask ReadAsync(long diskOffset, byte[] block, int offset, int count, + CancellationToken cancellationToken) + => ReadAsync(diskOffset, block.AsMemory(offset, count), cancellationToken); + + public override async ValueTask ReadAsync(long diskOffset, Memory block, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + int count = PrepareSourceRead(diskOffset, block.Length); + int read = await _stream!.ReadAsync(block[..count], cancellationToken).ConfigureAwait(false); + return CheckRead(read, count); + } + + private int PrepareSourceRead(long diskOffset, int count) + { + long position = diskOffset - Start; + _stream!.Position = position; + return (int)Math.Min(count, Length - position); + } + + private static int CheckRead(int read, int count) + { + if (read == 0 && count != 0) + { + throw new EndOfStreamException( + "A source file was truncated while reading the image." + ); + } + + return read; + } + + public override void DisposeReadState() + { + if (_stream != null) + { + _stream.Dispose(); + _stream = null; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + DisposeReadState(); + } + } + } +} diff --git a/Library/DiscUtils.Udf/UdfImageWriter.cs b/Library/DiscUtils.Udf/UdfImageWriter.cs new file mode 100644 index 000000000..7374466cf --- /dev/null +++ b/Library/DiscUtils.Udf/UdfImageWriter.cs @@ -0,0 +1,374 @@ +// SPDX-License-Identifier: MIT +#nullable enable +using System; +using System.IO; +using System.Linq; +using System.Text; +using DiscUtils.Streams; + +namespace DiscUtils.Udf; + +// ECMA-167, third edition, parts 3 and 4; UDF 2.01 mastered physical partition. +// Offsets are relative to each descriptor, including its 16-byte tag. +internal sealed class UdfImageWriter +{ + private const int BlockSize = 2048; + private const ushort Revision = 0x0201; + private const long MaximumExtentLength = (1L << 30) - BlockSize; + private readonly string _volume; + private readonly string _volumeSet; + private readonly DateTime _time; + + public UdfImageWriter(string volume, string volumeSet, DateTime time) + { + _volume = volume; + _volumeSet = volumeSet; + _time = time.ToUniversalTime(); + } + + public static byte[] VolumeRecognition() + { + var bytes = new byte[3 * BlockSize]; + string[] identifiers = { "BEA01", "NSR03", "TEA01" }; + for (int i = 0; i < identifiers.Length; i++) + { + Ascii(bytes, i * BlockSize + 1, identifiers[i]); + bytes[i * BlockSize + 6] = 1; + } + + return bytes; + } + + public byte[] Anchor(uint location) + { + var bytes = new byte[512]; + U32(bytes, 16, 16 * BlockSize); + U32(bytes, 20, 257); + U32(bytes, 24, 16 * BlockSize); + U32(bytes, 28, 273); + return Tag(bytes, 2, location); + } + + public byte[] PrimaryVolume(uint location) + { + var bytes = new byte[512]; + U32(bytes, 16, 1); + DString(bytes, 24, 32, _volume); + U16(bytes, 56, 1); + U16(bytes, 58, 1); + U16(bytes, 60, 2); + U16(bytes, 62, 3); + U32(bytes, 64, 1); + U32(bytes, 68, 1); + DString(bytes, 72, 128, _volumeSet); + Charset(bytes, 200); + Charset(bytes, 264); + Entity(bytes, 344, "*DiscUtils"); + Timestamp(bytes, 376); + Entity(bytes, 388, "*DiscUtils"); + U16(bytes, 488, 1); + return Tag(bytes, 1, location); + } + + public byte[] ImplementationVolume(uint location) + { + var bytes = new byte[512]; + U32(bytes, 16, 2); + Entity(bytes, 20, "*UDF LV Info"); + U16(bytes, 44, Revision); + Charset(bytes, 52); + DString(bytes, 116, 128, _volume); + Entity(bytes, 352, "*DiscUtils"); + return Tag(bytes, 4, location); + } + + public byte[] Partition(uint location, uint start, uint length) + { + var bytes = new byte[512]; + U32(bytes, 16, 3); + U16(bytes, 20, 1); + Entity(bytes, 24, "+NSR03"); + U32(bytes, 184, 1); // Read-only. + U32(bytes, 188, start); + U32(bytes, 192, length); + Entity(bytes, 196, "*DiscUtils"); + return Tag(bytes, 5, location); + } + + public byte[] LogicalVolume(uint location) + { + var bytes = new byte[446]; + U32(bytes, 16, 4); + Charset(bytes, 20); + DString(bytes, 84, 128, _volume); + U32(bytes, 212, BlockSize); + Domain(bytes, 216); + LongAd(bytes, 248, 2 * BlockSize, 0); + U32(bytes, 264, 6); + U32(bytes, 268, 1); + Entity(bytes, 272, "*DiscUtils"); + U32(bytes, 432, 2 * BlockSize); + U32(bytes, 436, 289); + bytes[440] = 1; // Type-1 physical partition map. + bytes[441] = 6; + U16(bytes, 442, 1); // Volume sequence number. + return Tag(bytes, 6, location); + } + + public byte[] UnallocatedSpace(uint location) + { + var bytes = new byte[24]; + U32(bytes, 16, 5); + return Tag(bytes, 7, location); + } + + public static byte[] Terminator(uint location) + { + return Tag(new byte[512], 8, location); + } + + public byte[] Integrity(uint partitionLength, ulong nextId, uint files, uint directories) + { + var bytes = new byte[134]; + Timestamp(bytes, 16); + U32(bytes, 28, 1); // Closed. + U64(bytes, 40, nextId); + U32(bytes, 72, 1); + U32(bytes, 76, 46); + U32(bytes, 84, partitionLength); + Entity(bytes, 88, "*DiscUtils"); + U32(bytes, 120, files); + U32(bytes, 124, directories); + U16(bytes, 128, Revision); + U16(bytes, 130, Revision); + U16(bytes, 132, Revision); + return Tag(bytes, 9, 289); + } + + public byte[] FileSet(uint rootIcb) + { + var bytes = new byte[512]; + Timestamp(bytes, 16); + U16(bytes, 28, 3); + U16(bytes, 30, 3); + U32(bytes, 32, 1); + U32(bytes, 36, 1); + Charset(bytes, 48); + DString(bytes, 112, 128, _volume); + Charset(bytes, 240); + DString(bytes, 304, 32, _volume); + LongAd(bytes, 400, BlockSize, rootIcb); + Domain(bytes, 416); + return Tag(bytes, 256, 0); + } + + public byte[] FileEntry( + uint icb, + uint dataBlock, + long length, + bool directory, + ushort links, + ulong uniqueId + ) + { + int allocationCount = checked( + (int)((length + MaximumExtentLength - 1) / MaximumExtentLength) + ); + var bytes = new byte[176 + allocationCount * 8]; + U16(bytes, 20, 4); // ICB strategy type. + U16(bytes, 24, 1); // Maximum ICB entries. + bytes[27] = directory ? (byte)4 : (byte)5; + U32(bytes, 36, uint.MaxValue); + U32(bytes, 40, uint.MaxValue); + U32(bytes, 44, directory ? 0x14a5u : 0x1084u); + U16(bytes, 48, links); + U64(bytes, 56, checked((ulong)length)); + U64(bytes, 64, checked((ulong)((length + BlockSize - 1) / BlockSize))); + Timestamp(bytes, 72); + Timestamp(bytes, 84); + Timestamp(bytes, 96); + U32(bytes, 108, 1); + Entity(bytes, 128, "*DiscUtils"); + U64(bytes, 160, uniqueId); + U32(bytes, 172, checked((uint)(allocationCount * 8))); + long remaining = length; + for (int i = 0; i < allocationCount; i++) + { + uint part = (uint)Math.Min(remaining, MaximumExtentLength); + U32(bytes, 176 + i * 8, part); + U32(bytes, 180 + i * 8, dataBlock); + remaining -= part; + dataBlock = checked(dataBlock + (part + BlockSize - 1) / BlockSize); + } + + return Tag(bytes, 261, icb); + } + + public int FileIdentifier( + byte[] output, + int offset, + uint location, + string name, + uint icb, + ulong uniqueId, + bool directory, + bool parent + ) + { + byte[] encoded = parent ? new byte[0] : EncodeName(name); + var bytes = new byte[(38 + encoded.Length + 3) / 4 * 4]; + U16(bytes, 16, 1); + bytes[18] = (byte)((directory ? 2 : 0) | (parent ? 8 : 0)); + bytes[19] = checked((byte)encoded.Length); + LongAd(bytes, 20, BlockSize, icb); + U32(bytes, 32, (uint)uniqueId); + Array.Copy(encoded, 0, bytes, 38, encoded.Length); + Tag(bytes, 257, location); + Array.Copy(bytes, 0, output, offset, bytes.Length); + return bytes.Length; + } + + public static byte[] EncodeName(string value) + { + if (value == null) + { + throw new ArgumentNullException(nameof(value)); + } + + bool wide = value.Any(character => character > 255); + Encoding encoding = wide + ? new UnicodeEncoding(true, false, true) + : Encoding.GetEncoding("ISO-8859-1"); + byte[] name = encoding.GetBytes(value); + if (name.Length > 254) + { + throw new ArgumentException( + "The UDF name exceeds 255 encoded bytes.", + nameof(value) + ); + } + + var encoded = new byte[name.Length + 1]; + encoded[0] = wide ? (byte)16 : (byte)8; + Array.Copy(name, 0, encoded, 1, name.Length); + return encoded; + } + + public static byte[] EncodeDString(string value, int size) + { + var bytes = new byte[size]; + if (string.IsNullOrEmpty(value)) + { + if (value == null) + { + throw new ArgumentNullException(nameof(value)); + } + + return bytes; + } + + byte[] encoded = EncodeName(value); + if (encoded.Length >= size) + { + throw new ArgumentException( + "The identifier exceeds its UDF field length.", + nameof(value) + ); + } + + Array.Copy(encoded, bytes, encoded.Length); + bytes[size - 1] = checked((byte)encoded.Length); + return bytes; + } + + private static byte[] Tag(byte[] bytes, ushort id, uint location) + { + U16(bytes, 0, id); + U16(bytes, 2, 3); // ECMA-167 third edition. + U16(bytes, 6, 1); + ushort crc = 0; + for (int i = 16; i < bytes.Length; i++) + { + crc ^= (ushort)(bytes[i] << 8); + for (int bit = 0; bit < 8; bit++) + { + crc = (ushort)((crc & 0x8000) != 0 ? (crc << 1) ^ 0x1021 : crc << 1); + } + } + + U16(bytes, 8, crc); + U16(bytes, 10, checked((ushort)(bytes.Length - 16))); + U32(bytes, 12, location); + int checksum = 0; + for (int i = 0; i < 16; i++) + { + if (i != 4) + { + checksum += bytes[i]; + } + } + + bytes[4] = (byte)checksum; + return bytes; + } + + private void Timestamp(byte[] bytes, int offset) + { + U16(bytes, offset, 0x1000); // Local-time type with UTC offset zero. + U16(bytes, offset + 2, checked((ushort)_time.Year)); + bytes[offset + 4] = (byte)_time.Month; + bytes[offset + 5] = (byte)_time.Day; + bytes[offset + 6] = (byte)_time.Hour; + bytes[offset + 7] = (byte)_time.Minute; + bytes[offset + 8] = (byte)_time.Second; + bytes[offset + 9] = (byte)(_time.Millisecond / 10); + } + + private static void Charset(byte[] bytes, int offset) + { + Ascii(bytes, offset + 1, "OSTA Compressed Unicode"); + } + + private static void Entity(byte[] bytes, int offset, string id) + { + Ascii(bytes, offset + 1, id); + } + + private static void Domain(byte[] bytes, int offset) + { + Entity(bytes, offset, "*OSTA UDF Compliant"); + U16(bytes, offset + 24, Revision); + bytes[offset + 26] = 3; // Hard and soft write protection. + } + + private static void LongAd(byte[] bytes, int offset, uint length, uint location) + { + U32(bytes, offset, length); + U32(bytes, offset + 4, location); + } + + private static void DString(byte[] bytes, int offset, int size, string value) + { + Array.Copy(EncodeDString(value, size), 0, bytes, offset, size); + } + + private static void Ascii(byte[] bytes, int offset, string value) + { + Encoding.ASCII.GetBytes(value, 0, value.Length, bytes, offset); + } + + private static void U16(byte[] bytes, int offset, ushort value) + { + EndianUtilities.WriteBytesLittleEndian(value, bytes.AsSpan(offset)); + } + + private static void U32(byte[] bytes, int offset, uint value) + { + EndianUtilities.WriteBytesLittleEndian(value, bytes.AsSpan(offset)); + } + + private static void U64(byte[] bytes, int offset, ulong value) + { + EndianUtilities.WriteBytesLittleEndian(value, bytes.AsSpan(offset)); + } +} diff --git a/README.md b/README.md index 5e1d0b90a..47a1a30c0 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ These tables describe implemented API capabilities, not maturity ratings or supp - **Create**: build a new image, filesystem or structured file. - **Modify**: change an existing instance through format-aware APIs. For virtual disks this includes writing virtual sector content; it does not imply arbitrary metadata editing, resizing or chain merging. -Creation and modification are distinct: ISO, SquashFS and XVA can be built as new images while existing images remain read-only. Writable operations also require appropriate backing-stream access. Raw byte writes forwarded by a wrapper do not establish structured format-editing support. +Creation and modification are distinct: ISO, UDF, SquashFS and XVA can be built as new images while existing images remain read-only. Writable operations also require appropriate backing-stream access. Raw byte writes forwarded by a wrapper do not establish structured format-editing support. ### Filesystems @@ -57,7 +57,7 @@ Filesystem constructors normally expect a stream beginning at the filesystem, su | [ISO 9660 / Joliet](Library/DiscUtils.Iso9660/README.md) | Yes | Yes | No | CDReader also supports Rock Ridge; CDBuilder builds new images, including bootable-image scenarios | | [NTFS](Library/DiscUtils.Ntfs/README.md) | Yes | Yes | Yes | Includes alternate data streams, security descriptors, reparse points and allocation metadata | | [SquashFS](Library/DiscUtils.SquashFs/README.md) | Yes | Yes | No | Separate reader and builder; zlib included, other codecs require callbacks | -| [UDF](Library/DiscUtils.Udf/README.md) | Yes | No | No | Size/used/free-space properties are not implemented | +| [UDF](Library/DiscUtils.Udf/README.md) | Yes | UDF 2.01 | No | Virtual image builder; size/used/free-space reader properties are not implemented | | [XFS](Library/DiscUtils.Xfs/README.md) | Yes | No | No | Unix metadata and allocation extents | ### Archive filesystem views diff --git a/Tests/LibraryTests/Udf/UdfBuilderApiTest.cs b/Tests/LibraryTests/Udf/UdfBuilderApiTest.cs new file mode 100644 index 000000000..c68855732 --- /dev/null +++ b/Tests/LibraryTests/Udf/UdfBuilderApiTest.cs @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: MIT +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using DiscUtils.Streams; +using DiscUtils.Streams.Compatibility; +using DiscUtils.Udf; +using Xunit; + +namespace LibraryTests.Udf; + +public class UdfBuilderApiTest +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SpanAndArrayReadsPreserveExtentBoundaries(bool useSpan) + { + using var source = new MemoryStream(); + source.Write(new byte[] { 1, 2, 3 }, 0, 3); + var builder = new UdfBuilder(); + builder.AddFile("file", 3, () => source); + using var image = builder.Build(); + long start = PayloadStart(image, 3); + image.Position = start; + Assert.Equal(1, image.ReadByte()); + source.Position = 3; + source.Write(new byte[] { 4, 5, 6 }, 0, 3); + image.Position = start; + var bytes = Enumerable.Repeat((byte)255, 10).ToArray(); + int read = useSpan ? image.Read(bytes.AsSpan(1, 8)) : image.Read(bytes, 1, 8); + Assert.Equal(8, read); + Assert.Equal(new byte[] { 255, 1, 2, 3, 0, 0, 0, 0, 0, 255 }, bytes); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task AsyncReadsUseSourceAsyncIoAndPreserveExtentBoundaries(bool useMemory) + { + int opens = 0; + using var source = new AsyncOnlyStream(new byte[] { 1, 2, 3 }); + var builder = new UdfBuilder(); + builder.AddFile("file", 3, () => { opens++; return source; }); + using var image = await builder.BuildAsync(CancellationToken.None); + Assert.Equal(0, opens); + long start = PayloadStart(image, 3); + image.Position = start; + var bytes = Enumerable.Repeat((byte)255, 10).ToArray(); + using var cancellation = new CancellationTokenSource(); + Assert.Equal(1, await image.ReadAsync(bytes, 0, 1, cancellation.Token)); + source.Append(new byte[] { 4, 5, 6 }); + image.Position = start; + bytes[0] = 255; + int read = useMemory + ? await image.ReadAsync(bytes.AsMemory(1, 8), cancellation.Token) + : await image.ReadAsync(bytes, 1, 8, cancellation.Token); + Assert.Equal(8, read); + Assert.Equal(new byte[] { 255, 1, 2, 3, 0, 0, 0, 0, 0, 255 }, bytes); + Assert.Equal(1, opens); + Assert.Equal(2, source.AsyncReads); + Assert.Equal(cancellation.Token, source.LastToken); + Assert.True(source.Disposed); // Leaving the payload closes its read state. + } + + [Fact] + public async Task AsyncMaterializationUsesSourceAsyncIo() + { + using var source = new AsyncOnlyStream(new byte[] { 1, 2, 3 }); + var builder = new UdfBuilder(); + builder.AddFile("file", 3, () => source); + using var output = new MemoryStream(); + await builder.BuildAsync(output, CancellationToken.None); + Assert.Equal(1, source.AsyncReads); + Assert.True(source.Disposed); + output.Position = 0; + using var reader = new UdfReader(output); + using var file = reader.OpenFile("file", FileMode.Open); + var bytes = new byte[3]; + Assert.Equal(3, file.Read(bytes, 0, 3)); + Assert.Equal(new byte[] { 1, 2, 3 }, bytes); + } + + [Fact] + public async Task AsyncBuildHonorsCancellationBeforeConstructingMetadata() + { + var builder = new UdfBuilder(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync( + () => builder.BuildAsync(cancellation.Token)); + using var output = new MemoryStream(); + await Assert.ThrowsAnyAsync( + () => builder.BuildAsync(output, cancellation.Token)); + Assert.Equal(0, output.Length); + } + + [Fact] + public async Task AsyncPayloadReadHonorsCancellation() + { + using var source = new AsyncOnlyStream(new byte[] { 1, 2, 3 }); + var builder = new UdfBuilder(); + builder.AddFile("file", 3, () => source); + using var image = builder.Build(); + image.Position = PayloadStart(image, 3); + long position = image.Position; + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + await Assert.ThrowsAnyAsync( + () => image.ReadAsync(new byte[3], 0, 3, cancellation.Token)); + Assert.Equal(position, image.Position); + Assert.Equal(0, source.AsyncReads); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UnexpectedTruncationIsReportedAcrossReadApis(bool useAsync) + { + using var source = new MemoryStream(); + source.Write(new byte[] { 1, 2, 3 }, 0, 3); + var builder = new UdfBuilder(); + builder.AddFile("file", 3, () => source); + using var image = builder.Build(); + image.Position = PayloadStart(image, 3); + Assert.Equal(1, image.ReadByte()); + source.SetLength(1); + if (useAsync) + { + await Assert.ThrowsAsync( + () => image.ReadAsync(new byte[1], 0, 1)); + } + else + { + Assert.Throws(() => image.Read(new byte[1].AsSpan())); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void InvalidSourceIsDisposedEvenWhenLengthAccessThrows(bool throwOnLength) + { + using var source = new InvalidSourceStream(throwOnLength); + var builder = new UdfBuilder(); + builder.AddFile("file", 3, () => source); + using var image = builder.Build(); + image.Position = PayloadStart(image, 3); + Assert.Throws(() => image.ReadByte()); + Assert.True(source.Disposed); + } + + [Fact] + public void BuildsHaveIndependentPositionsAndOwnedSources() + { + var sources = new System.Collections.Generic.List(); + var builder = new UdfBuilder(); + builder.AddFile("file", 3, () => + { + var source = new AsyncOnlyStream(new byte[] { 1, 2, 3 }, allowSync: true); + sources.Add(source); + return source; + }); + using var first = builder.Build(); + using var second = builder.Build(); + first.Position = PayloadStart(first, 3); + second.Position = PayloadStart(second, 3) + 1; + Assert.Equal(1, first.ReadByte()); + Assert.Equal(2, second.ReadByte()); + Assert.Equal(2, first.ReadByte()); + Assert.Equal(2, sources.Count); + first.Dispose(); + Assert.True(sources[0].Disposed); + Assert.False(sources[1].Disposed); + Assert.Equal(3, second.ReadByte()); + } + + [Fact] + public void SourcePathChangesAreRejectedBeforePayloadReads() + { + string path = Path.GetTempFileName(); + try + { + System.IO.File.WriteAllBytes(path, new byte[] { 1, 2, 3 }); + var builder = new UdfBuilder(); + builder.AddFile("file", path); + using var image = builder.Build(); + System.IO.File.WriteAllBytes(path, new byte[] { 4, 5, 6 }); + System.IO.File.SetLastWriteTimeUtc(path, System.IO.File.GetLastWriteTimeUtc(path).AddMinutes(1)); + image.Position = PayloadStart(image, 3); + Assert.Throws(() => image.ReadByte()); + } + finally + { + System.IO.File.Delete(path); + } + } + + private static long PayloadStart(Stream image, long length) + => ((SparseStream)image).Extents.Single(extent => extent.Length == length).Start; + + private sealed class InvalidSourceStream : MemoryStream + { + private readonly bool _throwOnLength; + public InvalidSourceStream(bool throwOnLength) : base(new byte[2]) + => _throwOnLength = throwOnLength; + + public bool Disposed { get; private set; } + public override long Length => _throwOnLength ? throw new IOException("Length unavailable.") : base.Length; + protected override void Dispose(bool disposing) + { + Disposed = true; + base.Dispose(disposing); + } + } + + private sealed class AsyncOnlyStream : ReadOnlyCompatibilityStream + { + private readonly MemoryStream _source; + private readonly bool _allowSync; + public AsyncOnlyStream(byte[] content, bool allowSync = false) + { + _source = new MemoryStream(); + _source.Write(content, 0, content.Length); + _source.Position = 0; + _allowSync = allowSync; + } + + public void Append(byte[] content) + { + long position = _source.Position; + _source.Position = _source.Length; + _source.Write(content, 0, content.Length); + _source.Position = position; + } + + public int AsyncReads { get; private set; } + public CancellationToken LastToken { get; private set; } + public bool Disposed { get; private set; } + public override bool CanRead => true; + public override bool CanSeek => true; + public override long Length => _source.Length; + public override long Position { get => _source.Position; set => _source.Position = value; } + public override long Seek(long offset, SeekOrigin origin) => _source.Seek(offset, origin); + public override int Read(byte[] buffer, int offset, int count) + => _allowSync ? _source.Read(buffer, offset, count) : throw new InvalidOperationException("Synchronous payload read."); + public override int Read(Span buffer) + => _allowSync ? _source.Read(buffer) : throw new InvalidOperationException("Synchronous payload read."); + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + AsyncReads++; + LastToken = cancellationToken; + return _source.ReadAsync(buffer, cancellationToken); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + Disposed = true; + _source.Dispose(); + } + base.Dispose(disposing); + } + } +} diff --git a/Tests/LibraryTests/Udf/UdfBuilderTest.cs b/Tests/LibraryTests/Udf/UdfBuilderTest.cs new file mode 100644 index 000000000..f4e5ec9f2 --- /dev/null +++ b/Tests/LibraryTests/Udf/UdfBuilderTest.cs @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: MIT +using System; +using System.IO; +using System.Linq; +using System.Text; +using DiscUtils.Streams; +using DiscUtils.Udf; +using Xunit; + +namespace LibraryTests.Udf; + +public class UdfBuilderTest +{ + [Fact] + public void BuildPreservesTreeNamesAndBytes() + { + var builder = new UdfBuilder { VolumeIdentifier = "TEST" }; + builder.AddDirectory("BDMV/EMPTY"); + builder.AddFile("BDMV/index.bdmv", new byte[] { 1, 2, 3, 4 }); + builder.AddFile("字幕/繁體中文.txt", Encoding.UTF8.GetBytes("字幕")); + builder.AddFile("empty", new byte[0]); + using (Stream image = builder.Build()) + using (var reader = new UdfReader(image)) + { + Assert.Equal("TEST", reader.VolumeLabel); + Assert.True(reader.DirectoryExists("BDMV/EMPTY".Replace('/', '\\'))); + using ( + Stream file = reader.OpenFile( + "BDMV/index.bdmv".Replace('/', '\\'), + FileMode.Open + ) + ) + { + Assert.Equal(new byte[] { 1, 2, 3, 4 }, ReadAll(file)); + } + + using ( + Stream file = reader.OpenFile( + "字幕/繁體中文.txt".Replace('/', '\\'), + FileMode.Open + ) + ) + { + Assert.Equal("字幕", Encoding.UTF8.GetString(ReadAll(file))); + } + + Assert.Equal(0, reader.GetFileLength("empty")); + } + } + + [Fact] + public void BuildDoesNotOpenPayloadAndSupportsLargeFileSeeks() + { + int opens = 0; + long length = (81L << 30) + 123; + var builder = new UdfBuilder(); + builder.AddFile( + "BDMV/STREAM/00001.m2ts", + length, + () => + { + opens++; + return new PatternStream(length); + } + ); + using (Stream image = builder.Build()) + { + Assert.Equal(0, opens); + Assert.True(image.Length > length); + using (var reader = new UdfReader(image)) + { + string path = "BDMV/STREAM/00001.m2ts".Replace('/', '\\'); + Assert.Equal(length, reader.GetFileLength(path)); + Assert.Equal(0, opens); + using (Stream file = reader.OpenFile(path, FileMode.Open)) + { + foreach ( + long offset in new long[] { 0, (1L << 30) - 2050, 5L << 30, length - 7 } + ) + { + file.Position = offset; + var bytes = new byte[(int)Math.Min(4096, length - offset)]; + Assert.Equal(bytes.Length, file.Read(bytes, 0, bytes.Length)); + Assert.Equal( + Enumerable + .Range(0, bytes.Length) + .Select(i => PatternStream.At(offset + i)), + bytes + ); + } + + Assert.Equal(-1, file.ReadByte()); + } + } + } + } + + [Fact] + public void DirectorySpansMultipleBlocks() + { + var builder = new UdfBuilder(); + for (int i = 0; i < 200; i++) + { + builder.AddFile( + "BDMV/PLAYLIST/" + i.ToString("00000") + ".mpls", + new byte[] { (byte)i } + ); + } + + using (Stream image = builder.Build()) + using (var reader = new UdfReader(image)) + { + string directory = "BDMV/PLAYLIST".Replace('/', '\\'); + Assert.Equal(200, reader.GetFiles(directory).Count()); + using ( + Stream file = reader.OpenFile(directory + "\\" + "00199.mpls", FileMode.Open) + ) + { + Assert.Equal(199, file.ReadByte()); + } + } + } + + [Fact] + public void BuildsAreIndependentSnapshots() + { + var builder = new UdfBuilder(); + builder.AddFile("first", new byte[] { 1 }); + using (Stream first = builder.Build()) + { + builder.AddFile("second", new byte[] { 2 }); + using (Stream second = builder.Build()) + using (var firstReader = new UdfReader(first)) + using (var secondReader = new UdfReader(second)) + { + Assert.False(firstReader.FileExists("second")); + Assert.True(secondReader.FileExists("second")); + using (Stream file = firstReader.OpenFile("first", FileMode.Open)) + { + Assert.Equal(1, file.ReadByte()); + } + } + } + } + + [Fact] + public void RejectsInvalidPathsAndUnsupportedLengths() + { + var builder = new UdfBuilder(); + foreach ( + string path in new[] { "", "/absolute", "../escape", "a/../b", "a//b", "a\0b" } + ) + { + Assert.Throws(() => builder.AddFile(path, new byte[0])); + } + + Assert.Throws(() => + builder.AddFile(new string('a', 255), new byte[0]) + ); + Assert.Throws(() => + builder.AddFile("large", 234 * ((1L << 30) - 2048) + 1, () => Stream.Null) + ); + Assert.Throws(() => + builder.AddFile("negative", -1, () => Stream.Null) + ); + builder.AddFile("existing", new byte[0]); + Assert.Throws(() => builder.AddDirectory("existing/child")); + Assert.Throws(() => builder.AddFile("existing", new byte[0])); + } + + [Fact] + public void RejectsChangedSourceLength() + { + var builder = new UdfBuilder(); + builder.AddFile("file", 10, () => new MemoryStream(new byte[9])); + using (Stream image = builder.Build()) + using (var reader = new UdfReader(image)) + using (Stream file = reader.OpenFile("file", FileMode.Open)) + { + Assert.Throws(() => file.ReadByte()); + } + } + + [Fact] + public void SourceGrowthCannotOverwriteImagePadding() + { + var payload = new MemoryStream(); + payload.Write(new byte[] { 1, 2, 3, 4 }, 0, 4); + var builder = new UdfBuilder(); + builder.AddFile("file", 4, () => payload); + using (Stream image = builder.Build()) + using (var reader = new UdfReader(image)) + using (Stream file = reader.OpenFile("file", FileMode.Open)) + { + Assert.Equal(1, file.ReadByte()); + long start = image.Position - 1; + payload.Position = 4; + payload.Write(new byte[] { 5, 6, 7, 8 }, 0, 4); + image.Position = start; + var bytes = new byte[8]; + Assert.Equal(8, image.Read(bytes, 0, bytes.Length)); + Assert.Equal(new byte[] { 1, 2, 3, 4, 0, 0, 0, 0 }, bytes); + } + } + + private static byte[] ReadAll(Stream stream) + { + using (var output = new MemoryStream()) + { + stream.CopyTo(output); + return output.ToArray(); + } + } + + private sealed class PatternStream : Stream + { + private readonly long _length; + + public PatternStream(long length) + { + _length = length; + } + + public static byte At(long position) + { + return (byte)((position >> 8) ^ position); + } + + public override bool CanRead + { + get { return true; } + } + public override bool CanSeek + { + get { return true; } + } + public override bool CanWrite + { + get { return false; } + } + public override long Length + { + get { return _length; } + } + public override long Position { get; set; } + + public override int Read(byte[] buffer, int offset, int count) + { + int available = (int)Math.Min(count, Length - Position); + for (int i = 0; i < available; i++) + { + buffer[offset + i] = At(Position++); + } + return available; + } + + public override long Seek(long offset, SeekOrigin origin) + { + Position = + ( + origin == SeekOrigin.Begin ? 0 + : origin == SeekOrigin.Current ? Position + : Length + ) + offset; + return Position; + } + + public override void Flush() { } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + } +}