diff --git a/.latexmkrc b/.latexmkrc index a6cf24a..b010326 100644 --- a/.latexmkrc +++ b/.latexmkrc @@ -2,3 +2,6 @@ $pdf_mode = 4; # lualatex $bibtex_use = 2; # biber $out_dir = out; $clean_ext .= ' %R.run.xml %R.bbl %R.bcf %R-blx.bib'; + +# Never stop for a prompt: a failing listing would otherwise loop forever. +$lualatex = 'lualatex -interaction=nonstopmode -halt-on-error %O %S'; diff --git a/appendix/appendix-code.tex b/appendix/appendix-code.tex index 7ffae9b..678b31e 100644 --- a/appendix/appendix-code.tex +++ b/appendix/appendix-code.tex @@ -1,10 +1,109 @@ -% TODO: Code-Ausschnitte — je in center-Umgebung mit \captionof{listing}{...} + \inputminted +\newcommand{\codelisting}[4]{% + \par\medskip\noindent + \begingroup + \captionsetup{type=listing,hypcap=false}% + \captionof{listing}{#1}% + \label{#2}% + \endgroup + \nopagebreak + \inputminted[ + fontsize=\scriptsize, + linenos, + breaklines, + breakanywhere, + breaksymbolleft={}, + frame=single, + framesep=1.5mm, + tabsize=4, + numbersep=3mm, + ]{#3}{figures/code/#4}% + \par\medskip +} -% Reihenfolge gemäß Kapiteln: -% s3-settings-registration.cs, s3-documents-client.cs, -% documents-page-model.cs, document-view-model.cs, documents-table-body.cshtml, -% document-type-mapping.cs, document-types.resx, -% document-search.cs, pagination-continuation-token.cs, -% zip-download-action.cs, presigned-url.cs, url-file-parsing.cs, -% org-slug.cs, resolve-customer-prefix.cs, rename-customer-folder.cs, -% authorization-policy.cs, documents-service-tests.cs +Die folgenden Ausschnitte stammen aus dem Houston-Repository und geben den Stand +zum Ende des Projektzeitraums wieder. Sie sind auf die jeweils besprochene Stelle +gekürzt; Auslassungen sind mit \texttt{// \ldots} gekennzeichnet. + +\subsection{Anbindung des Objektspeichers} + +\codelisting{Registrierung des S3-Clients als benannter Dienst, einschließlich der +Korrektur des \texttt{x-amz-copy-source}-Headers}{lst:s3-client-registration}{csharp}{s3-client-registration.cs} + +\codelisting{Konfigurationsobjekte für Endpunkt, Zugangsdaten und Bucket}{lst:s3-settings}{csharp}{s3-settings.cs} + +\codelisting{Autorisierung des Dokumentenbereichs in \texttt{Program.cs}}{lst:auth-policy}{csharp}{auth-policy.cs} + +\clearpage +\subsection{Typisierte Pfade} + +\codelisting{\texttt{DocumentKey} — vollständiger Objektschlüssel einschließlich +Kundenordner, nur über eine validierende Fabrikmethode erzeugbar}{lst:document-key}{csharp}{document-key.cs} + +\codelisting{\texttt{DocumentId} — die in URLs verwendete Kennung, die den +Kundenordner bewusst nicht enthält}{lst:document-id}{csharp}{document-id.cs} + +\codelisting{\texttt{DocumentName} — Ableitung von Anzeigename, PDF- und +Verknüpfungseigenschaft aus der Dateiendung}{lst:document-name}{csharp}{document-name.cs} + +\clearpage +\subsection{Dokumenttypen und Symbole} + +\codelisting{Dokumenttypen und Ableitung des Ordnernamens aus der +Übersetzungsressource}{lst:document-type}{csharp}{document-type.cs} + +\codelisting{Einbindung der Typsymbole als CSS-Maske, damit sie die Themenfarbe +übernehmen}{lst:type-icon-mask}{scss}{type-icon-mask.scss} + +\clearpage +\subsection{Auflistung, Suche und Blätterfunktion} + +\codelisting{Durchlaufen aller Antwortseiten über den Fortsetzungs-Token}{lst:pagination}{csharp}{pagination-continuation-token.cs} + +\codelisting{Auflisten der Dokumente eines Kunden mit Einstiegspunkt für die +Folgeseite}{lst:list-documents}{csharp}{list-documents.cs} + +\codelisting{Anwendungsseitige Filterung nach Suchbegriff und +Dokumenttyp}{lst:document-search}{csharp}{document-search.cs} + +\codelisting{Tabellenkörper der Dokumentenübersicht}{lst:documents-table}{html}{documents-table-body.cshtml} + +\clearpage +\subsection{Download, Vorschau und Freigabe} + +\codelisting{Erzeugung vorsignierter URLs und der zugehörigen +\texttt{Content-Disposition}-Kopfzeilen}{lst:presigned-url}{csharp}{presigned-url.cs} + +\codelisting{Streamen des ZIP-Archivs ohne Zwischenpufferung}{lst:zip-archive}{csharp}{zip-archive.cs} + +\codelisting{Handler des ZIP-Downloads einschließlich der gezielten Freigabe +synchroner Schreibvorgänge}{lst:zip-handler}{csharp}{zip-download-handler.cs} + +\codelisting{Anonyme Landeseite für Freigabelinks}{lst:share-page}{csharp}{share-page.cs} + +\clearpage +\subsection{Verknüpfungsdateien} + +\codelisting{Der eigene INI-Parser}{lst:ini-parser}{csharp}{ini-parser.cs} + +\codelisting{Auswertung einer \texttt{.url}-Datei mit Prüfung des +Zielschemas}{lst:url-file}{csharp}{url-file-parsing.cs} + +\clearpage +\subsection{Kundenordner} + +\codelisting{Bildung des kanonischen Ordnernamens aus dem +Efecte-Firmennamen}{lst:org-slug}{csharp}{org-slug.cs} + +\codelisting{Dreistufiger Lookup des Kundenordners}{lst:resolve-customer-prefix}{csharp}{resolve-customer-prefix.cs} + +\codelisting{Umbenennen eines Kundenordners — die in Abschnitt~\ref{sec:race-conditions} +beschriebene Stelle}{lst:rename-folder}{csharp}{rename-customer-folder.cs} + +\clearpage +\subsection{Tests} + +\codelisting{Testdoppel des Objektspeichers: geschriebene Objekte werden für +spätere Abfragen sichtbar}{lst:tests-empty-bucket}{csharp}{tests-empty-bucket.cs} + +\codelisting{Test gegen Pfadmanipulation — die Kennung wird bewusst unter Umgehung +der Validierung erzeugt}{lst:test-path-traversal}{csharp}{test-path-traversal.cs} diff --git a/figures/code/auth-policy.cs b/figures/code/auth-policy.cs new file mode 100644 index 0000000..d72d10e --- /dev/null +++ b/figures/code/auth-policy.cs @@ -0,0 +1,25 @@ +// Houston/Auth/HoustonRoles.cs +public static readonly string DocumentsView = "Documents.View"; + +// Houston/Program.cs +var viewDocumentsPolicy = "CanViewDocuments"; + +builder.Services.AddAuthorization(o => +{ + o.FallbackPolicy = new AuthorizationPolicyBuilder().RequireAuthenticatedUser().Build(); + // ... + o.AddPolicy(viewDocumentsPolicy, policy => policy.RequireRole(HoustonRoles.DocumentsView)); +}); + +builder.Services.AddRazorPages(o => +{ + // ... + o.Conventions.AllowAnonymousToPage("/Document/Share"); + o.Conventions.AuthorizeFolder("/Document", viewDocumentsPolicy); + o.Conventions.AuthorizePage("/Documents", viewDocumentsPolicy); +}); + +// Houston/Services registrations +builder.Services.AddOptions().BindConfiguration("Documents"); +builder.Services.AddDocumentsClient(); +builder.Services.AddScoped(); diff --git a/figures/code/document-id.cs b/figures/code/document-id.cs new file mode 100644 index 0000000..9615758 --- /dev/null +++ b/figures/code/document-id.cs @@ -0,0 +1,42 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text; +using Microsoft.AspNetCore.WebUtilities; + +namespace Houston.Model.Documents; + +/// +/// Public URL id of a customer document. +/// +/// +/// The identifier used in URLs (e.g. /document/{id}/download or /document/{id}/share) +/// is the URL base64 encoding of the document path without the customer/organization +/// folder. For an object stored at OrgX/TypeY/FileZ.pdf the id therefore encodes +/// TypeY/FileZ.pdf. The organization part is never part of the id, so it can only ever be +/// derived server side from the authenticated user. +/// +public readonly record struct DocumentId(string Value) +{ + public static DocumentId Encode(DocumentRelativePath relativePath) + => new(WebEncoders.Base64UrlEncode(Encoding.UTF8.GetBytes(relativePath.ToString()))); + + public bool TryDecode([NotNullWhen(true)] out DocumentRelativePath? relativePath) + { + relativePath = null; + + if (String.IsNullOrEmpty(Value)) + return false; + + try + { + var decoded = Encoding.UTF8.GetString(WebEncoders.Base64UrlDecode(Value)); + return DocumentRelativePath.TryFromS3Key(decoded, out relativePath); + } + catch (FormatException) + { + relativePath = null; + return false; + } + } + + public override string ToString() => Value; +} \ No newline at end of file diff --git a/figures/code/document-key.cs b/figures/code/document-key.cs new file mode 100644 index 0000000..764fd61 --- /dev/null +++ b/figures/code/document-key.cs @@ -0,0 +1,41 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text; + +namespace Houston.Model.Documents; + +/// +/// Full S3 object key of a customer document, including the customer/organization prefix. +/// +public readonly record struct DocumentKey(OrgSlug Org, DocumentType? Type, DocumentName Name) +{ + public static bool TryFromS3Key(string key, [NotNullWhen(true)] out DocumentKey? result) + { + result = null; + + if (key.EndsWith('/')) return false; + + var parts = key.Split('/', StringSplitOptions.RemoveEmptyEntries); + + if (parts.Contains("..")) return false; + + result = parts switch + { + [var org, var name] when OrgSlug.TryFromFolderName(org, out var slug) + => new(slug.Value, null, new(name)), + [var org, var type, var name] when OrgSlug.TryFromFolderName(org, out var slug) + => new(slug.Value, DocumentTypeExtensions.ParseStorageFolderName(type), new(name)), + _ => null, + }; + + return result is not null; + } + + public override string ToString() + { + StringBuilder result = new(); + result.Append(Org.ToString()); + if (Type is { } type) result.Append(type.StorageFolderName).Append('/'); + result.Append(Name); + return result.ToString(); + } +} \ No newline at end of file diff --git a/figures/code/document-name.cs b/figures/code/document-name.cs new file mode 100644 index 0000000..f8e9fec --- /dev/null +++ b/figures/code/document-name.cs @@ -0,0 +1,27 @@ +namespace Houston.Model.Documents; + +public readonly record struct DocumentName(string Value) +{ + public bool Matches(string pattern) + { + return Value.Contains(pattern, StringComparison.CurrentCultureIgnoreCase); + } + + /// + /// Whether this document is a PDF and therefore eligible for the inline modal preview. + /// + public bool IsPdf => Value.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase); + + /// + /// Whether this is an external document link (a .URL file). + /// + public bool IsLink => Value.EndsWith(".url", StringComparison.OrdinalIgnoreCase); + + /// + /// File display name. + /// + public string DisplayName => IsLink ? Value[..^".url".Length] : Value; + + /// + public override string ToString() => Value; +} \ No newline at end of file diff --git a/figures/code/document-search.cs b/figures/code/document-search.cs new file mode 100644 index 0000000..1717624 --- /dev/null +++ b/figures/code/document-search.cs @@ -0,0 +1,39 @@ +private async IAsyncEnumerable SearchDocumentsAsync( + OrgSlug org, + string? searchTerm, + DocumentTypeSet documentTypes, + bool includeUntypedDocuments, + DocumentRelativePath? startAfter, + [EnumeratorCancellation] CancellationToken cancellationToken) +{ + var query = searchTerm?.Trim(); + + await foreach (var doc in ListDocumentsAsync(org, startAfter, cancellationToken)) + { + if (!doc.Type.MatchesFilter(documentTypes, includeUntypedDocuments)) + continue; + + if (!String.IsNullOrEmpty(query) && !doc.Name.Matches(query)) + continue; + + yield return doc; + } +} + +// Houston/Model/Documents/DocumentName.cs +public bool Matches(string pattern) +{ + return Value.Contains(pattern, StringComparison.CurrentCultureIgnoreCase); +} + +// Houston/Model/Documents/DocumentType.cs +public bool MatchesFilter(DocumentTypeSet documentTypes, bool includeUntypedDocuments) +{ + if (documentTypes.IsEmpty && !includeUntypedDocuments) + return true; + + if (documentType is null) + return includeUntypedDocuments; + + return documentTypes.Types.Contains(documentType.Value); +} diff --git a/figures/code/document-type.cs b/figures/code/document-type.cs new file mode 100644 index 0000000..87125ca --- /dev/null +++ b/figures/code/document-type.cs @@ -0,0 +1,42 @@ +using System.Globalization; +using Houston.Resources; + +namespace Houston.Model.Documents; + +public enum DocumentType +{ + ServiceProtocol, + AcceptanceDocuments, + SlaReportsTicketProcessing, + MonitoringReports, + SecurityAssessments, + BillingData, + ContractDocuments, +} + +public static class DocumentTypeExtensions +{ + public const string OtherDocumentsDisplayName = "Sonstige Dokumente"; + public const string OtherDocumentsValue = "OtherDocuments"; + + public static string? FromStorageFolderName(string storageFolderName) => + APITranslation.ResourceManager.GetString(storageFolderName, CultureInfo.InvariantCulture); + + public static DocumentType? ParseStorageFolderName(string storageFolderName) + { + var type = FromStorageFolderName(storageFolderName); + if (!Enum.TryParse(type, ignoreCase: true, out var variant)) + return null; + + return variant; + } + + extension(DocumentType documentType) + { + /// + /// The name of the S3 subfolder that holds documents of this type, i.e. the German translation. + /// + public string StorageFolderName => + APITranslation.ResourceManager.GetString(documentType.ToString(), CultureInfo.InvariantCulture) + ?? throw new NotImplementedException($"Missing translation for DocumentType '{documentType}'."); + } diff --git a/figures/code/documents-table-body.cshtml b/figures/code/documents-table-body.cshtml new file mode 100644 index 0000000..d9b4ee2 --- /dev/null +++ b/figures/code/documents-table-body.cshtml @@ -0,0 +1,65 @@ + +@foreach (var file in Model.Files) +{ + + + @if (Documents.IsDownloadable(file.Name)) + { +
+ +
+ } + + + + + @file.Name.DisplayName + +
+ @if (file.Name.IsPdf) + { + + } + else + { + + } + + @if (file.Name.IsLink) + { + + + + } + else + { + + + + } +
+ + +} diff --git a/figures/code/ini-parser.cs b/figures/code/ini-parser.cs new file mode 100644 index 0000000..8d80481 --- /dev/null +++ b/figures/code/ini-parser.cs @@ -0,0 +1,49 @@ +namespace Houston.Services; + +public class IniParser +{ + public async Task ParseAsync(TextReader reader, CancellationToken cancellationToken) + { + var sections = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var current = new Dictionary(StringComparer.OrdinalIgnoreCase); + sections[String.Empty] = current; + + while (await reader.ReadLineAsync(cancellationToken) is { } line) + { + var trimmed = line.Trim(); + + // Skip blank lines and comments + if (trimmed.Length == 0 || trimmed[0] is ';' or '#') + continue; + + if (trimmed.StartsWith('[') && trimmed.EndsWith(']')) + { + var name = trimmed[1..^1].Trim(); + if (!sections.TryGetValue(name, out current!)) + sections[name] = current = new Dictionary(StringComparer.OrdinalIgnoreCase); + continue; + } + + var separator = trimmed.IndexOf('='); + if (separator < 0) + continue; + + var key = trimmed[..separator].Trim(); + if (key.Length == 0) + continue; + + // Last value wins for duplicate keys within a section. + current[key] = trimmed[(separator + 1)..].Trim(); + } + + return new(sections); + } + + /// + /// Parses INI content from a string. + /// + public Task ParseAsync(string content, CancellationToken cancellationToken) + { + using var reader = new StringReader(content); + return ParseAsync(reader, cancellationToken); + } diff --git a/figures/code/list-documents.cs b/figures/code/list-documents.cs new file mode 100644 index 0000000..1386d4d --- /dev/null +++ b/figures/code/list-documents.cs @@ -0,0 +1,25 @@ +private async IAsyncEnumerable ListDocumentsAsync( + OrgSlug org, + DocumentRelativePath? startAfter, + [EnumeratorCancellation] CancellationToken cancellationToken) +{ + var isFirstRequest = true; + var requestBuilder = () => + { + var r = new ListObjectsV2Request + { + Prefix = org.ToString(), + StartAfter = isFirstRequest && startAfter is { } doc + ? new DocumentKey(org, doc.Type, doc.Name).ToString() + : null, + }; + + isFirstRequest = false; + + return r; + }; + + await foreach (var obj in EnumerateObjectsAsync(requestBuilder, cancellationToken)) + if (DocumentKey.TryFromS3Key(obj.Key, out var key)) + yield return key.Value; +} diff --git a/figures/code/org-slug.cs b/figures/code/org-slug.cs new file mode 100644 index 0000000..905a746 --- /dev/null +++ b/figures/code/org-slug.cs @@ -0,0 +1,112 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text; + +namespace Houston.Model.Documents; + +public readonly record struct OrgSlug +{ + public const int MaxLength = 128; + + private const string CharactersToReplace = @"/\{}^%`][""><~#|"; + + private const char ReplacementCharacter = '-'; + + /// + /// Characters that must not start or end a folder name, because they are either invisible or + /// break path semantics on the client side. + /// + private static readonly char[] UntrimmableEdgeCharacters = [' ', '.', ReplacementCharacter]; + + public string Value { get; } + + private OrgSlug(string value) => Value = value; + + public static bool TryFromS3Key(string key, [NotNullWhen(true)] out OrgSlug? result) + { + result = null; + + if (!key.EndsWith('/')) return false; + + var parts = key.Split('/', StringSplitOptions.RemoveEmptyEntries); + + return parts is [var name] && TryFromFolderName(name, out result); + } + + public static bool TryFromFolderName(string name, [NotNullWhen(true)] out OrgSlug? result) + { + result = null; + + if (String.IsNullOrWhiteSpace(name)) return false; + if (name.Contains('/')) return false; + if (name == "..") return false; + + result = new OrgSlug(name); + return true; + } + + /// + /// Builds the canonical folder name for an organization. + /// + /// + /// false if the name contains nothing that survives the normalization, in which case + /// there is no folder name Houston could safely claim. + /// + public static bool TryFrom(OrgName name, [NotNullWhen(true)] out OrgSlug? result) + { + result = null; + + var slug = Normalize(name.Value, MaxLength); + if (slug.Length == 0) + return false; + + result = new OrgSlug(slug); + return true; + } + + public static bool TryFrom(OrgName name, OrgId id, [NotNullWhen(true)] out OrgSlug? result) + { + result = null; + + var normalizedId = Normalize(id.Value, MaxLength); + if (normalizedId.Length == 0) + return false; + + // The id is what makes the folder name unique, so the name is what gets truncated. + var suffix = $" ({normalizedId})"; + if (suffix.Length >= MaxLength) + return false; + + var slug = Normalize(name.Value, MaxLength - suffix.Length); + + result = new OrgSlug(slug.Length == 0 ? normalizedId : slug + suffix); + return true; + } + + private static string Normalize(string value, int maxLength) + { + var builder = new StringBuilder(Math.Min(value.Length, maxLength)); + + foreach (var character in value.Trim().TakeWhile(c => builder.Length != maxLength)) + { + builder.Append(Char.IsControl(character) || CharactersToReplace.Contains(character) + ? ReplacementCharacter + : character); + } + + // Truncating can uncover a trailing space or dot, so trimming happens afterwards. + return builder.ToString().Trim().Trim(UntrimmableEdgeCharacters).Trim(); + } + + /// + /// S3 prefix of the folder, including the trailing slash. + /// + public override string ToString() + { + // A default(OrgSlug) would resolve to the bucket root and expose every customer's + // documents, so it must never be used as a prefix. + if (String.IsNullOrEmpty(Value)) + throw new InvalidOperationException("An uninitialized OrgSlug cannot be used as an S3 prefix."); + + return Value + '/'; + } +} diff --git a/figures/code/pagination-continuation-token.cs b/figures/code/pagination-continuation-token.cs new file mode 100644 index 0000000..1723dcf --- /dev/null +++ b/figures/code/pagination-continuation-token.cs @@ -0,0 +1,28 @@ +private async IAsyncEnumerable EnumerateResponsesAsync( + Func requestBuilder, + [EnumeratorCancellation] CancellationToken cancellationToken) +{ + string? continuationToken = null; + do + { + var request = requestBuilder(); + request.ContinuationToken = continuationToken; + request.BucketName = _settings.S3.Bucket; + + var response = await s3.ListObjectsV2Async(request, cancellationToken); + yield return response; + + continuationToken = response.IsTruncated == true ? response.NextContinuationToken : null; + } while (continuationToken is not null); +} + +private async IAsyncEnumerable EnumerateObjectsAsync( + Func requestBuilder, + [EnumeratorCancellation] CancellationToken cancellationToken) +{ + await foreach (var response in EnumerateResponsesAsync(requestBuilder, cancellationToken)) + { + foreach (var obj in response.S3Objects ?? []) + yield return obj; + } +} diff --git a/figures/code/presigned-url.cs b/figures/code/presigned-url.cs new file mode 100644 index 0000000..efbdf78 --- /dev/null +++ b/figures/code/presigned-url.cs @@ -0,0 +1,39 @@ +private async Task GetPreSignedUrlAsync(DocumentKey key, bool inline, CancellationToken cancellationToken) +{ + if (!await ObjectExistsAsync(key, cancellationToken)) + return null; + + var request = new GetPreSignedUrlRequest + { + BucketName = _settings.S3.Bucket, + Key = key.ToString(), + Verb = HttpVerb.GET, + Expires = DateTime.UtcNow.Add(DownloadUrlLifetime), + ResponseHeaderOverrides = + { + ContentDisposition = inline + ? InlineContentDisposition(key.Name.ToString()) + : AttachmentContentDisposition(key.Name.ToString()), + }, + }; + + return await s3.GetPreSignedURLAsync(request); +} + +private static string AttachmentContentDisposition(string fileName) +{ + var header = new ContentDispositionHeaderValue("attachment"); + header.SetHttpFileName(fileName); + return header.ToString(); +} + +/// +/// Builds a Content-Disposition header that lets the browser render the document inline +/// while preserving the file name, using RFC 5987 encoding so non ASCII file names survive. +/// +private static string InlineContentDisposition(string fileName) +{ + var asciiFallback = new string(fileName.Select(c => c is >= ' ' and < (char)127 and not '"' ? c : '_').ToArray()); + var encoded = Uri.EscapeDataString(fileName); + return $"inline; filename=\"{asciiFallback}\"; filename*=UTF-8''{encoded}"; +} diff --git a/figures/code/rename-customer-folder.cs b/figures/code/rename-customer-folder.cs new file mode 100644 index 0000000..730d59c --- /dev/null +++ b/figures/code/rename-customer-folder.cs @@ -0,0 +1,54 @@ +/// +/// todo: this function can currently cause dataloss. see #10070 +/// +/// success +private async Task RenameCustomerFolderAsync( + OrgSlug source, OrgSlug destination, OrgId owner, CancellationToken cancellationToken) +{ + if (source == destination) + return true; + + if (!await CanClaimCustomerFolderAsync(destination, owner, cancellationToken)) + return false; + + var sourcePrefix = source.ToString(); + var destinationPrefix = destination.ToString(); + + var sourceKeys = await ListKeysAsync(sourcePrefix, cancellationToken); + + // The marker of the customer folder carries the efecte-org-id, so it is copied last and + // deleted first: the destination only becomes resolvable once it is complete. Everything + // below the customer folder keeps its relative key, including subfolders Houston does not + // know, so no document ever ends up somewhere else. + var documentKeys = sourceKeys.Where(key => key != sourcePrefix).ToList(); + var markerKeys = sourceKeys.Where(key => key == sourcePrefix).ToList(); + var copiedKeys = new List(sourceKeys.Count); + + try + { + foreach (var key in documentKeys.Concat(markerKeys)) + { + var destinationKey = destinationPrefix + key[sourcePrefix.Length..]; + await CopyObjectAsync(key, destinationKey, cancellationToken); + copiedKeys.Add(destinationKey); + } + } + catch (AmazonS3Exception exception) + { + ExceptionlessClient.Default.SubmitLog( + $"Could not move customer folder '{source}' to '{destination}': {exception.Message}. " + + "The folder stays where it is.", LogLevel.Error); + + // Leftover copies would block every later attempt, because the destination is only + // claimed while it is empty. Cleaning them up keeps the move retryable. + await TryDeleteObjectsAsync(copiedKeys, cancellationToken); + return false; + } + + // From here on the destination is authoritative. A failure while deleting only leaves + // garbage behind, the customer already sees all of his documents under the new name. + await TryDeleteObjectsAsync(documentKeys, cancellationToken); + await TryDeleteObjectsAsync(markerKeys, cancellationToken); + + return true; +} diff --git a/figures/code/resolve-customer-prefix.cs b/figures/code/resolve-customer-prefix.cs new file mode 100644 index 0000000..14b047a --- /dev/null +++ b/figures/code/resolve-customer-prefix.cs @@ -0,0 +1,64 @@ +private async Task GetCustomerFolderAsync(OrgName name, OrgId id, CancellationToken cancellationToken) +{ + if (!OrgSlug.TryFrom(name, out var canonicalSlug) || !OrgSlug.TryFrom(name, id, out var collisionSlug)) + { + ExceptionlessClient.Default.SubmitLog($"Efecte name '{name}' of organization {id} does not yield a usable folder name.", LogLevel.Warn); + return null; + } + + var (prettySlug, uniqueSlug) = (canonicalSlug.Value, collisionSlug.Value); + + // Fast path: pretty slug exists & meta-id matches => return slug(name) + var prettyState = await InspectCustomerFolderAsync(prettySlug, id, cancellationToken); + if (prettyState is CustomerFolderState.Owned) + return prettySlug; + + // Conflict resolution: pretty slug exists & meta-id does not match => return slug(name,id) + var uniqueState = await InspectCustomerFolderAsync(uniqueSlug, id, cancellationToken); + if (uniqueState is CustomerFolderState.Owned) + return uniqueSlug; + + // State cleanup: + // Either the folder does not have the name it should have or doesn't exist, so the bucket has to be scanned. And + // if necessary and possible, renamed such that the next lookup hits one of the expected slugs. + + OrgSlug? availableSlug = + prettyState is CustomerFolderState.Missing ? prettySlug + : uniqueState is CustomerFolderState.Missing ? uniqueSlug + : null; + + var currentSlug = await SearchCustomerPrefixAsync(id, cancellationToken); + + if (currentSlug is null) + { + if (availableSlug is not { } available) + { + ExceptionlessClient.Default.SubmitLog( + $"Organization {id} has no customer folder and neither " + + $"'{prettySlug}' nor '{uniqueSlug}' is available.", LogLevel.Error); + return null; + } + + if (!await CreateCustomerFolderAsync(available, id, cancellationToken)) + return null; + + if (await InspectCustomerFolderAsync(available, id, cancellationToken) is not CustomerFolderState.Owned) + { + ExceptionlessClient.Default.SubmitLog( + $"Organization {id} has no customer folder and " + + $"'{available}' could not be created.", LogLevel.Error); + return null; + } + + return available; + } + + if (availableSlug is null || availableSlug.Value == currentSlug.Value) + return currentSlug; + + // A failed move leaves every document under the name it already had, so that name is what + // gets handed back. The next lookup simply tries the move again. + return await RenameCustomerFolderAsync(currentSlug.Value, availableSlug.Value, id, cancellationToken) + ? availableSlug + : currentSlug; +} diff --git a/figures/code/s3-client-registration.cs b/figures/code/s3-client-registration.cs new file mode 100644 index 0000000..39bf0fc --- /dev/null +++ b/figures/code/s3-client-registration.cs @@ -0,0 +1,38 @@ +using Amazon.Runtime; +using Amazon.S3; +using Houston.Settings; +using Microsoft.Extensions.Options; + +namespace Houston.Extensions; + +public static class DocumentsExtensions +{ + extension(IServiceCollection services) + { + public IServiceCollection AddDocumentsClient() + { + return services.AddKeyedScoped("Documents", (services, _) => + { + var settings = services.GetRequiredService>(); + var client = new AmazonS3Client( + new BasicAWSCredentials(settings.Value.S3.Key, settings.Value.S3.Secret), + new AmazonS3Config { ServiceURL = settings.Value.S3.Endpoint }); + + // The SDK percent-encodes the separators in x-amz-copy-source ("bucket%2Fkey%2Fdoc.pdf"). + // AWS decodes that again, our S3 splits bucket and key on the first literal slash and ends up + // without a key ("Invalid copy source object key"), so the separators are restored here. The + // event runs before the signer, hence the corrected value is the one that gets signed. + client.BeforeRequestEvent += (_, args) => + { + if (args is WebServiceRequestEventArgs { Headers: { } headers } + && headers.TryGetValue("x-amz-copy-source", out var copySource) + && copySource.Contains("%2F", StringComparison.Ordinal)) + { + headers["x-amz-copy-source"] = copySource.Replace("%2F", "/", StringComparison.Ordinal); + } + }; + + return client; + }); + } + } diff --git a/figures/code/s3-settings.cs b/figures/code/s3-settings.cs new file mode 100644 index 0000000..7ddc2bb --- /dev/null +++ b/figures/code/s3-settings.cs @@ -0,0 +1,16 @@ +namespace Houston.Settings; + +public class DocumentsSettings +{ + public required S3Settings S3 { get; set; } +} + +namespace Houston.Settings; + +public class S3Settings +{ + public required string Endpoint { get; set; } + public required string Key { get; set; } + public required string Secret { get; set; } + public required string Bucket { get; set; } +} \ No newline at end of file diff --git a/figures/code/share-page.cs b/figures/code/share-page.cs new file mode 100644 index 0000000..41d5302 --- /dev/null +++ b/figures/code/share-page.cs @@ -0,0 +1,61 @@ +using Houston.Model.Documents; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http.Extensions; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.RazorPages; + +namespace Houston.Pages.Document; + +/// +/// Anonymous landing page for shared document links. It exposes only metadata derived from the +/// document id and never fetches or renders file contents. +/// +public class Share : PageModel +{ + [BindProperty(SupportsGet = true)] + public string Id { get; set; } = String.Empty; + + public string Title { get; private set; } = "Dokument"; + + public string ImageUrl { get; private set; } = String.Empty; + + public string ExplorerUrl { get; private set; } = String.Empty; + + public IActionResult OnGet() + { + var docId = new DocumentId(Id); + + if (!docId.TryDecode(out var relativePath)) + return NotFound(); + + var type = relativePath.Value.Type; + var name = relativePath.Value.Name; + var iconPath = type.PreviewPath; + + Title = name.DisplayName; + ImageUrl = AbsoluteUrl(iconPath); + ExplorerUrl = AbsoluteUrl("/documents", BuildExplorerQuery(type, name), $"#{Documents.DocumentViewId(docId)}"); + + return Page(); + } + + private static QueryString BuildExplorerQuery(DocumentType? type, DocumentName name) + { + return new QueryBuilder { + { "q", name.DisplayName }, + { "types", type.Name } + }.ToQueryString(); + } + + private string AbsoluteUrl(string path, QueryString query = default, string? fragment = null) + { + return UriHelper.BuildAbsolute( + Request.Scheme, + Request.Host, + Request.PathBase, + path, + query, + fragment: fragment is null ? default : new FragmentString(fragment) + ); + } +} \ No newline at end of file diff --git a/figures/code/test-path-traversal.cs b/figures/code/test-path-traversal.cs new file mode 100644 index 0000000..bcd9b4b --- /dev/null +++ b/figures/code/test-path-traversal.cs @@ -0,0 +1,12 @@ +[Theory] +[InlineData("../Kunde2/secret.pdf")] +[InlineData("sub/../../Kunde2/secret.pdf")] +public void DocumentIdTryParseRejectsPathTraversal(string relativePath) +{ + Assert.False(new DocumentId(UnsafeDocumentIdValue(relativePath)).TryDecode(out _)); +} + + +// Test-Hilfsfunktion: erzeugt bewusst eine ID, die die Validierung umgeht. +private static string UnsafeDocumentIdValue(string relativePath) + => WebEncoders.Base64UrlEncode(Encoding.UTF8.GetBytes(relativePath)); diff --git a/figures/code/tests-empty-bucket.cs b/figures/code/tests-empty-bucket.cs new file mode 100644 index 0000000..143fe66 --- /dev/null +++ b/figures/code/tests-empty-bucket.cs @@ -0,0 +1,44 @@ + +/// +/// A bucket without a single object: every lookup answers with a 404 and every listing is empty +/// until a test puts something in via , +/// or . +/// +private static IAmazonS3 EmptyBucket() +{ + var s3 = Substitute.For(); + + // Objects written during the test become visible to later lookups, just like in a real bucket. + var writtenObjects = new Dictionary(StringComparer.Ordinal); + + s3.GetObjectMetadataAsync(Arg.Any(), Arg.Any()) + .Returns(call => + { + var key = call.Arg().Key; + + if (!writtenObjects.TryGetValue(key, out var owner)) + throw new AmazonS3Exception("not found") { StatusCode = System.Net.HttpStatusCode.NotFound }; + + return owner is null ? new GetObjectMetadataResponse() : MetadataResponse(owner); + }); + + s3.ListObjectsV2Async(Arg.Any(), Arg.Any()) + .Returns(_ => new ListObjectsV2Response { IsTruncated = false }); + + s3.CopyObjectAsync(Arg.Any(), Arg.Any()) + .Returns(_ => new CopyObjectResponse()); + + s3.PutObjectAsync(Arg.Any(), Arg.Any()) + .Returns(call => + { + var request = call.Arg(); + writtenObjects[request.Key] = request.Metadata["efecte-org-id"]; + return new PutObjectResponse(); + }); + + s3.DeleteObjectsAsync(Arg.Any(), Arg.Any()) + .Returns(_ => new DeleteObjectsResponse()); + + return s3; +} + diff --git a/figures/code/type-icon-mask.scss b/figures/code/type-icon-mask.scss new file mode 100644 index 0000000..2b2f459 --- /dev/null +++ b/figures/code/type-icon-mask.scss @@ -0,0 +1,28 @@ +.doc-type-icon { + display: inline-flex; + width: 1.25em; + height: 1.25em; + vertical-align: middle; + align-items: center; + justify-content: center; +} + +@each $name in ( + 'service-protokoll', + 'abnahme-dokumente', + 'sla-reports', + 'monitoring-reports', + 'security-assessments', + 'abrechnungsdaten', + 'vertragsunterlagen', + 'default', + 'ms-sharepoint', + 'ms-onedrive', + 'ms-teams', +) { + .doc-type-icon-#{$name} { + background-color: currentColor; + mask-repeat: no-repeat; + mask-position: center; + mask-size: contain; + mask-image: url('/img/document-types/icons/#{$name}.svg'); diff --git a/figures/code/url-file-parsing.cs b/figures/code/url-file-parsing.cs new file mode 100644 index 0000000..8ad233c --- /dev/null +++ b/figures/code/url-file-parsing.cs @@ -0,0 +1,43 @@ +public async Task TryGetUrlFileIconClassAsync(DocumentKey doc, CancellationToken cancellationToken) +{ + var ini = await TryReadUrlFileAsync(doc, cancellationToken); + var iconFile = ini?.GetValue("InternetShortcut", "IconFile"); + + return String.IsNullOrEmpty(iconFile) ? null : iconFile; +} + +private async Task TryGetUrlFileTargetAsync(DocumentKey doc, CancellationToken cancellationToken) +{ + var ini = await TryReadUrlFileAsync(doc, cancellationToken); + var target = ini?.GetValue("InternetShortcut", "URL"); + + if (String.IsNullOrWhiteSpace(target)) + return null; + + if (!Uri.TryCreate(target, UriKind.Absolute, out var uri)) + return null; + + if (!(uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps)) + return null; + + return uri.ToString(); +} + +private async Task TryReadUrlFileAsync(DocumentKey doc, CancellationToken cancellationToken) +{ + try + { + using var response = await s3.GetObjectAsync(new GetObjectRequest + { + BucketName = _settings.S3.Bucket, + Key = doc.ToString(), + }, cancellationToken); + + using var reader = new StreamReader(response.ResponseStream); + return await iniParser.ParseAsync(reader, cancellationToken); + } + catch (AmazonS3Exception exception) when (exception.StatusCode == HttpStatusCode.NotFound) + { + return null; + } +} diff --git a/figures/code/zip-archive.cs b/figures/code/zip-archive.cs new file mode 100644 index 0000000..2247861 --- /dev/null +++ b/figures/code/zip-archive.cs @@ -0,0 +1,41 @@ +/// +/// Streams the prepared documents into a ZIP archive written directly onto +/// +public async Task WriteZipArchiveAsync( + OrgSlug orgSlug, IEnumerable documents, + Stream destination, CancellationToken cancellationToken) +{ + await using var zip = await ZipArchive.CreateAsync( + destination, ZipArchiveMode.Create, leaveOpen: true, + entryNameEncoding: null, cancellationToken); + + foreach (var path in documents) + { + var key = new DocumentKey(orgSlug, path.Type, path.Name); + + using var response = await TryGetObjectAsync(new GetObjectRequest + { + BucketName = _settings.S3.Bucket, + Key = key.ToString(), + }, cancellationToken); + + if (response is null) + continue; + + var entry = zip.CreateEntry(path.ToString(), CompressionLevel.Optimal); + await using var entryStream = await entry.OpenAsync(cancellationToken); + await response.ResponseStream.CopyToAsync(entryStream, cancellationToken); + } +} + +private async Task TryGetObjectAsync(GetObjectRequest request, CancellationToken cancellationToken) +{ + try + { + return await s3.GetObjectAsync(request, cancellationToken); + } + catch (AmazonS3Exception exception) when (exception.StatusCode == HttpStatusCode.NotFound) + { + return null; + } +} diff --git a/figures/code/zip-download-handler.cs b/figures/code/zip-download-handler.cs new file mode 100644 index 0000000..cf25282 --- /dev/null +++ b/figures/code/zip-download-handler.cs @@ -0,0 +1,42 @@ +public async Task OnPostDownloadZipAsync(CancellationToken cancellationToken) +{ + if (!OrgName.TryCreate(User.GetCompanyName(), out var orgName)) + return NotFound(); + + var fallback = RedirectToPage(new + { + q = Search, + types = TypeFilterSelection, + pageSize = PageSize, + pageToken = PageToken, + }); + + if (SelectedDocuments is null || SelectedDocuments.Count == 0) + return fallback; + + var relativePaths = SelectedDocuments + .SelectWhere(x => new DocumentId(x).TryDecode(out var y) ? y.Value : null) + .Where(x => IsDownloadable(x.Name)) + .ToList(); + + if (relativePaths.Count == 0) + return fallback; + + var orgSlug = await documents.ResolveCustomerPrefixAsync(OrgId, orgName.Value, cancellationToken); + if (orgSlug is null) + return fallback; + + // Zipping compresses each entry with a DeflateStream, which flushes its buffers synchronously + // when the entry is closed. Kestrel forbids synchronous response writes by default, so we opt + // in for this streamed response only. Nothing large is written synchronously - the document + // payloads are still copied with async IO. + var bodyControl = HttpContext.Features.Get(); + bodyControl?.AllowSynchronousIO = true; + + Response.ContentType = "application/zip"; + Response.Headers.ContentDisposition = $"attachment; filename=\"{DocumentsService.ZipDownloadFileName}\""; + + await documents.WriteZipArchiveAsync(orgSlug.Value, relativePaths, Response.Body, cancellationToken); + + return new EmptyResult(); +} diff --git a/shell.nix b/shell.nix index 2b506e3..f983c2f 100644 --- a/shell.nix +++ b/shell.nix @@ -1,17 +1,19 @@ -with import {}; let +let + # Pinned to the same nixpkgs revision as itc.componentware, where the minted + # toolchain is known to work. The channel version ships a latexminted that + # crashes on Python 3.14, which makes every code listing fail. + pkgs = import (builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/cc3f2ee0b3909e42334f34720ccac109a7e67068.tar.gz"; + sha256 = "sha256:0lz0yl9fmh5wfqp7j7rmcs8qjqr3bf1h5cy4rfdasbnbzh4k9j6x"; + }) {}; + fonts = [ - times-newer-roman + pkgs.times-newer-roman ]; - # This is not reccocgnised by latexmk, so we will have to put our - # font files in ~/.local/share/fonts - # fontsConf = makeFontsConf { - # fontDirectories = fonts; - # }; - # export FONTCONFIG_FILE="${fontsConf}" in - mkShell { + pkgs.mkShell { nativeBuildInputs = fonts; - packages = [ + packages = with pkgs; [ latexrun mermaid-cli times-newer-roman @@ -42,6 +44,6 @@ in # install fonts DEST=~/.local/share/fonts/TimesNewerRoman mkdir -p $DEST - cp -r ${toString times-newer-roman}/share/fonts/opentype "$DEST" + cp -rn ${toString pkgs.times-newer-roman}/share/fonts/opentype "$DEST" 2>/dev/null || true ''; }